2024-11-13 18:36:09,999 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 18:36:10,015 main DEBUG Took 0.013554 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 18:36:10,015 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 18:36:10,016 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 18:36:10,017 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 18:36:10,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,027 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 18:36:10,040 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,041 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,042 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,044 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,045 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,046 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,046 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,048 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,049 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,050 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,051 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,052 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,053 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,053 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,054 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,054 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:36:10,055 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,056 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 18:36:10,058 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:36:10,059 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 18:36:10,062 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 18:36:10,062 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 18:36:10,064 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 18:36:10,064 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 18:36:10,076 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 18:36:10,078 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 18:36:10,081 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 18:36:10,081 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 18:36:10,082 main DEBUG createAppenders(={Console}) 2024-11-13 18:36:10,082 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-13 18:36:10,083 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 18:36:10,083 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-13 18:36:10,084 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 18:36:10,084 main DEBUG OutputStream closed 2024-11-13 18:36:10,085 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 18:36:10,085 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 18:36:10,085 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-13 18:36:10,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 18:36:10,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 18:36:10,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 18:36:10,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 18:36:10,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 18:36:10,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 18:36:10,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 18:36:10,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 18:36:10,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 18:36:10,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 18:36:10,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 18:36:10,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 18:36:10,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 18:36:10,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 18:36:10,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 18:36:10,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 18:36:10,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 18:36:10,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 18:36:10,196 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 18:36:10,197 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-13 18:36:10,197 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 18:36:10,198 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-13T18:36:10,480 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687 2024-11-13 18:36:10,483 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 18:36:10,484 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T18:36:10,496 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-13T18:36:10,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T18:36:10,549 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/cluster_a648365b-8906-3617-8e5f-6b51b5e809af, deleteOnExit=true 2024-11-13T18:36:10,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T18:36:10,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/test.cache.data in system properties and HBase conf 2024-11-13T18:36:10,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T18:36:10,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/hadoop.log.dir in system properties and HBase conf 2024-11-13T18:36:10,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T18:36:10,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T18:36:10,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T18:36:10,675 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T18:36:10,799 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:36:10,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:36:10,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:36:10,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T18:36:10,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:36:10,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T18:36:10,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T18:36:10,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:36:10,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:36:10,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T18:36:10,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/nfs.dump.dir in system properties and HBase conf 2024-11-13T18:36:10,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/java.io.tmpdir in system properties and HBase conf 2024-11-13T18:36:10,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:36:10,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T18:36:10,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T18:36:12,052 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T18:36:12,133 INFO [Time-limited test {}] log.Log(170): Logging initialized @3040ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T18:36:12,224 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:36:12,296 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:36:12,320 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:36:12,320 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:36:12,322 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:36:12,340 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:36:12,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:36:12,344 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:36:12,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/java.io.tmpdir/jetty-localhost-42157-hadoop-hdfs-3_4_1-tests_jar-_-any-13023403502235531171/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:36:12,565 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42157} 2024-11-13T18:36:12,565 INFO [Time-limited test {}] server.Server(415): Started @3472ms 2024-11-13T18:36:13,016 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:36:13,025 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:36:13,026 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:36:13,027 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:36:13,027 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:36:13,028 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:36:13,029 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:36:13,149 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/java.io.tmpdir/jetty-localhost-41135-hadoop-hdfs-3_4_1-tests_jar-_-any-13244680818431906188/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:36:13,150 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:41135} 2024-11-13T18:36:13,150 INFO [Time-limited test {}] server.Server(415): Started @4058ms 2024-11-13T18:36:13,208 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:36:13,746 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/cluster_a648365b-8906-3617-8e5f-6b51b5e809af/data/data1/current/BP-941627117-172.17.0.3-1731522971541/current, will proceed with Du for space computation calculation, 2024-11-13T18:36:13,746 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/cluster_a648365b-8906-3617-8e5f-6b51b5e809af/data/data2/current/BP-941627117-172.17.0.3-1731522971541/current, will proceed with Du for space computation calculation, 2024-11-13T18:36:13,801 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:36:13,863 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687 2024-11-13T18:36:13,864 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x47c459ad96071430 with lease ID 0x22c7309c3ea7d7bf: Processing first storage report for DS-43627c6c-1ac2-4d78-b60b-b6c4203a554c from datanode DatanodeRegistration(127.0.0.1:42373, datanodeUuid=03a43742-4062-494e-8538-b44d5612c322, infoPort=33713, infoSecurePort=0, ipcPort=41465, storageInfo=lv=-57;cid=testClusterID;nsid=1188404576;c=1731522971541) 2024-11-13T18:36:13,866 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x47c459ad96071430 with lease ID 0x22c7309c3ea7d7bf: from storage DS-43627c6c-1ac2-4d78-b60b-b6c4203a554c node DatanodeRegistration(127.0.0.1:42373, datanodeUuid=03a43742-4062-494e-8538-b44d5612c322, infoPort=33713, infoSecurePort=0, ipcPort=41465, storageInfo=lv=-57;cid=testClusterID;nsid=1188404576;c=1731522971541), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T18:36:13,866 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x47c459ad96071430 with lease ID 0x22c7309c3ea7d7bf: Processing first storage report for DS-9c910967-b831-46b1-ba71-9d68bad6e9df from datanode DatanodeRegistration(127.0.0.1:42373, datanodeUuid=03a43742-4062-494e-8538-b44d5612c322, infoPort=33713, infoSecurePort=0, ipcPort=41465, storageInfo=lv=-57;cid=testClusterID;nsid=1188404576;c=1731522971541) 2024-11-13T18:36:13,867 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x47c459ad96071430 with lease ID 0x22c7309c3ea7d7bf: from storage DS-9c910967-b831-46b1-ba71-9d68bad6e9df node DatanodeRegistration(127.0.0.1:42373, datanodeUuid=03a43742-4062-494e-8538-b44d5612c322, infoPort=33713, infoSecurePort=0, ipcPort=41465, storageInfo=lv=-57;cid=testClusterID;nsid=1188404576;c=1731522971541), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:36:13,956 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/cluster_a648365b-8906-3617-8e5f-6b51b5e809af/zookeeper_0, clientPort=53425, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/cluster_a648365b-8906-3617-8e5f-6b51b5e809af/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/cluster_a648365b-8906-3617-8e5f-6b51b5e809af/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T18:36:13,967 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53425 2024-11-13T18:36:13,982 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:36:13,986 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:36:14,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:36:14,684 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 with version=8 2024-11-13T18:36:14,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/hbase-staging 2024-11-13T18:36:14,788 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T18:36:15,057 INFO [Time-limited test {}] client.ConnectionUtils(128): master/39e84130bbc9:0 server-side Connection retries=45 2024-11-13T18:36:15,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:36:15,071 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:36:15,077 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:36:15,077 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:36:15,077 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:36:15,220 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T18:36:15,297 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T18:36:15,311 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T18:36:15,316 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:36:15,344 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 19077 (auto-detected) 2024-11-13T18:36:15,345 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T18:36:15,366 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36105 2024-11-13T18:36:15,391 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36105 connecting to ZooKeeper ensemble=127.0.0.1:53425 2024-11-13T18:36:15,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361050x0, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:36:15,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36105-0x100ed62c5980000 connected 2024-11-13T18:36:15,459 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:36:15,462 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:36:15,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:36:15,478 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462, hbase.cluster.distributed=false 2024-11-13T18:36:15,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:36:15,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36105 2024-11-13T18:36:15,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36105 2024-11-13T18:36:15,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36105 2024-11-13T18:36:15,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36105 2024-11-13T18:36:15,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36105 2024-11-13T18:36:15,626 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39e84130bbc9:0 server-side Connection retries=45 2024-11-13T18:36:15,628 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:36:15,628 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:36:15,628 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:36:15,629 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:36:15,629 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:36:15,632 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:36:15,634 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:36:15,635 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35235 2024-11-13T18:36:15,637 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35235 connecting to ZooKeeper ensemble=127.0.0.1:53425 2024-11-13T18:36:15,638 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:36:15,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:36:15,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352350x0, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:36:15,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:352350x0, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:36:15,653 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35235-0x100ed62c5980001 connected 2024-11-13T18:36:15,654 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:36:15,663 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:36:15,665 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:36:15,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:36:15,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35235 2024-11-13T18:36:15,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35235 2024-11-13T18:36:15,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35235 2024-11-13T18:36:15,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35235 2024-11-13T18:36:15,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35235 2024-11-13T18:36:15,696 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;39e84130bbc9:36105 2024-11-13T18:36:15,697 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/39e84130bbc9,36105,1731522974847 2024-11-13T18:36:15,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:36:15,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:36:15,706 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/39e84130bbc9,36105,1731522974847 2024-11-13T18:36:15,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:36:15,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:15,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:15,735 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T18:36:15,736 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/39e84130bbc9,36105,1731522974847 from backup master directory 2024-11-13T18:36:15,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:36:15,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/39e84130bbc9,36105,1731522974847 2024-11-13T18:36:15,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:36:15,740 WARN [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:36:15,740 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=39e84130bbc9,36105,1731522974847 2024-11-13T18:36:15,743 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T18:36:15,745 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T18:36:15,818 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/hbase.id] with ID: 6b58678c-3cab-43db-b775-68cd0f8afd7f 2024-11-13T18:36:15,818 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/.tmp/hbase.id 2024-11-13T18:36:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:36:16,248 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/.tmp/hbase.id]:[hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/hbase.id] 2024-11-13T18:36:16,293 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:36:16,299 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T18:36:16,325 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-13T18:36:16,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:16,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:16,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:36:16,373 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:36:16,377 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T18:36:16,396 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:16,401 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:36:16,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:36:16,453 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store 2024-11-13T18:36:16,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:36:16,479 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T18:36:16,482 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:16,484 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:36:16,484 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:36:16,484 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:36:16,486 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:36:16,486 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:36:16,486 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:36:16,488 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731522976484Disabling compacts and flushes for region at 1731522976484Disabling writes for close at 1731522976486 (+2 ms)Writing region close event to WAL at 1731522976486Closed at 1731522976486 2024-11-13T18:36:16,491 WARN [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/.initializing 2024-11-13T18:36:16,491 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/WALs/39e84130bbc9,36105,1731522974847 2024-11-13T18:36:16,501 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:36:16,517 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39e84130bbc9%2C36105%2C1731522974847, suffix=, logDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/WALs/39e84130bbc9,36105,1731522974847, archiveDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/oldWALs, maxLogs=10 2024-11-13T18:36:16,541 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/WALs/39e84130bbc9,36105,1731522974847/39e84130bbc9%2C36105%2C1731522974847.1731522976522, exclude list is [], retry=0 2024-11-13T18:36:16,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42373,DS-43627c6c-1ac2-4d78-b60b-b6c4203a554c,DISK] 2024-11-13T18:36:16,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T18:36:16,599 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/WALs/39e84130bbc9,36105,1731522974847/39e84130bbc9%2C36105%2C1731522974847.1731522976522 2024-11-13T18:36:16,600 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33713:33713)] 2024-11-13T18:36:16,601 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:36:16,601 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:16,606 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,607 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,671 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T18:36:16,675 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:16,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:16,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T18:36:16,683 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:16,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:16,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T18:36:16,687 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:16,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:16,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,691 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T18:36:16,691 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:16,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:16,692 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,697 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,699 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,704 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,704 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,708 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:36:16,712 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:36:16,717 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:36:16,719 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67415126, jitterRate=0.004563659429550171}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:36:16,725 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731522976619Initializing all the Stores at 1731522976622 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522976622Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522976623 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522976623Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522976623Cleaning up temporary data from old regions at 1731522976705 (+82 ms)Region opened successfully at 1731522976724 (+19 ms) 2024-11-13T18:36:16,726 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T18:36:16,769 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d96b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39e84130bbc9/172.17.0.3:0 2024-11-13T18:36:16,808 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T18:36:16,820 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T18:36:16,820 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T18:36:16,823 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:36:16,825 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T18:36:16,830 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T18:36:16,830 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T18:36:16,856 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T18:36:16,867 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T18:36:16,869 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T18:36:16,871 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T18:36:16,872 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T18:36:16,874 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T18:36:16,876 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T18:36:16,879 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T18:36:16,881 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T18:36:16,882 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T18:36:16,884 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T18:36:16,902 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T18:36:16,904 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T18:36:16,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:36:16,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:36:16,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:16,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:16,911 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=39e84130bbc9,36105,1731522974847, sessionid=0x100ed62c5980000, setting cluster-up flag (Was=false) 2024-11-13T18:36:16,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:16,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:16,930 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T18:36:16,932 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39e84130bbc9,36105,1731522974847 2024-11-13T18:36:16,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:16,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:16,943 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T18:36:16,945 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39e84130bbc9,36105,1731522974847 2024-11-13T18:36:16,951 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T18:36:16,981 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(746): ClusterId : 6b58678c-3cab-43db-b775-68cd0f8afd7f 2024-11-13T18:36:16,983 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:36:16,989 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:36:16,989 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:36:16,994 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:36:16,995 DEBUG [RS:0;39e84130bbc9:35235 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1918cccd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39e84130bbc9/172.17.0.3:0 2024-11-13T18:36:17,016 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;39e84130bbc9:35235 2024-11-13T18:36:17,018 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:36:17,019 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:36:17,019 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:36:17,021 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(2659): reportForDuty to master=39e84130bbc9,36105,1731522974847 with port=35235, startcode=1731522975587 2024-11-13T18:36:17,033 DEBUG [RS:0;39e84130bbc9:35235 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:36:17,034 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:36:17,045 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T18:36:17,055 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T18:36:17,067 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47609, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:36:17,063 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 39e84130bbc9,36105,1731522974847 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T18:36:17,073 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/39e84130bbc9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:36:17,073 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36105 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:17,073 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/39e84130bbc9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:36:17,074 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/39e84130bbc9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:36:17,074 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/39e84130bbc9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:36:17,074 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/39e84130bbc9:0, corePoolSize=10, maxPoolSize=10 2024-11-13T18:36:17,074 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,075 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/39e84130bbc9:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:36:17,075 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,077 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731523007077 2024-11-13T18:36:17,079 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T18:36:17,081 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T18:36:17,081 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:36:17,082 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T18:36:17,085 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T18:36:17,086 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T18:36:17,086 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T18:36:17,086 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T18:36:17,088 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:17,088 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T18:36:17,087 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,090 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T18:36:17,091 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T18:36:17,092 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T18:36:17,094 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T18:36:17,094 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T18:36:17,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:36:17,097 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.large.0-1731522977096,5,FailOnTimeoutGroup] 2024-11-13T18:36:17,098 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.small.0-1731522977097,5,FailOnTimeoutGroup] 2024-11-13T18:36:17,098 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,098 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T18:36:17,100 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T18:36:17,100 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,100 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,100 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 2024-11-13T18:36:17,108 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T18:36:17,108 WARN [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T18:36:17,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:36:17,209 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(2659): reportForDuty to master=39e84130bbc9,36105,1731522974847 with port=35235, startcode=1731522975587 2024-11-13T18:36:17,212 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36105 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39e84130bbc9,35235,1731522975587 2024-11-13T18:36:17,214 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36105 {}] master.ServerManager(517): Registering regionserver=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:17,223 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 2024-11-13T18:36:17,223 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33577 2024-11-13T18:36:17,224 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:36:17,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:36:17,229 DEBUG [RS:0;39e84130bbc9:35235 {}] zookeeper.ZKUtil(111): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39e84130bbc9,35235,1731522975587 2024-11-13T18:36:17,229 WARN [RS:0;39e84130bbc9:35235 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:36:17,230 INFO [RS:0;39e84130bbc9:35235 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:36:17,230 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/WALs/39e84130bbc9,35235,1731522975587 2024-11-13T18:36:17,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39e84130bbc9,35235,1731522975587] 2024-11-13T18:36:17,262 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:36:17,280 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:36:17,286 INFO [RS:0;39e84130bbc9:35235 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:36:17,286 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,287 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:36:17,293 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:36:17,294 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,295 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,295 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,295 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,295 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,296 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,296 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39e84130bbc9:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:36:17,296 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,296 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,296 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,297 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,297 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,297 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:36:17,297 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39e84130bbc9:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:36:17,298 DEBUG [RS:0;39e84130bbc9:35235 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:36:17,299 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,300 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,300 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,300 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,300 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,300 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,35235,1731522975587-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:36:17,323 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:36:17,326 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,35235,1731522975587-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,326 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,326 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.Replication(171): 39e84130bbc9,35235,1731522975587 started 2024-11-13T18:36:17,348 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:17,349 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1482): Serving as 39e84130bbc9,35235,1731522975587, RpcServer on 39e84130bbc9/172.17.0.3:35235, sessionid=0x100ed62c5980001 2024-11-13T18:36:17,349 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:36:17,350 DEBUG [RS:0;39e84130bbc9:35235 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39e84130bbc9,35235,1731522975587 2024-11-13T18:36:17,350 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39e84130bbc9,35235,1731522975587' 2024-11-13T18:36:17,350 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:36:17,351 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:36:17,352 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:36:17,352 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:36:17,352 DEBUG [RS:0;39e84130bbc9:35235 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39e84130bbc9,35235,1731522975587 2024-11-13T18:36:17,352 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39e84130bbc9,35235,1731522975587' 2024-11-13T18:36:17,353 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:36:17,354 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:36:17,355 DEBUG [RS:0;39e84130bbc9:35235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:36:17,355 INFO [RS:0;39e84130bbc9:35235 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:36:17,355 INFO [RS:0;39e84130bbc9:35235 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:36:17,461 INFO [RS:0;39e84130bbc9:35235 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:36:17,464 INFO [RS:0;39e84130bbc9:35235 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39e84130bbc9%2C35235%2C1731522975587, suffix=, logDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/WALs/39e84130bbc9,35235,1731522975587, archiveDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/oldWALs, maxLogs=32 2024-11-13T18:36:17,482 DEBUG [RS:0;39e84130bbc9:35235 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/WALs/39e84130bbc9,35235,1731522975587/39e84130bbc9%2C35235%2C1731522975587.1731522977467, exclude list is [], retry=0 2024-11-13T18:36:17,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42373,DS-43627c6c-1ac2-4d78-b60b-b6c4203a554c,DISK] 2024-11-13T18:36:17,491 INFO [RS:0;39e84130bbc9:35235 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/WALs/39e84130bbc9,35235,1731522975587/39e84130bbc9%2C35235%2C1731522975587.1731522977467 2024-11-13T18:36:17,492 DEBUG [RS:0;39e84130bbc9:35235 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33713:33713)] 2024-11-13T18:36:17,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:17,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:36:17,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:36:17,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:17,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:17,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:36:17,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:36:17,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:17,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:17,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:36:17,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:36:17,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:17,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:17,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:36:17,533 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:36:17,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:17,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:17,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:36:17,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740 2024-11-13T18:36:17,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740 2024-11-13T18:36:17,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:36:17,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:36:17,540 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:36:17,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:36:17,547 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:36:17,548 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65696184, jitterRate=-0.021050572395324707}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:36:17,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731522977516Initializing all the Stores at 1731522977518 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522977518Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522977518Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522977518Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522977518Cleaning up temporary data from old regions at 1731522977540 (+22 ms)Region opened successfully at 1731522977550 (+10 ms) 2024-11-13T18:36:17,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:36:17,550 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:36:17,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:36:17,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:36:17,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:36:17,552 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:36:17,552 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522977550Disabling compacts and flushes for region at 1731522977550Disabling writes for close at 1731522977550Writing region close event to WAL at 1731522977551 (+1 ms)Closed at 1731522977552 (+1 ms) 2024-11-13T18:36:17,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:36:17,556 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T18:36:17,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:36:17,571 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:36:17,574 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T18:36:17,727 DEBUG [39e84130bbc9:36105 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T18:36:17,740 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:17,747 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39e84130bbc9,35235,1731522975587, state=OPENING 2024-11-13T18:36:17,752 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T18:36:17,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:17,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:36:17,755 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:36:17,755 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:36:17,757 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:36:17,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:36:17,935 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:36:17,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59875, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:36:17,951 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:36:17,951 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:36:17,952 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:36:17,956 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39e84130bbc9%2C35235%2C1731522975587.meta, suffix=.meta, logDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/WALs/39e84130bbc9,35235,1731522975587, archiveDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/oldWALs, maxLogs=32 2024-11-13T18:36:17,981 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/WALs/39e84130bbc9,35235,1731522975587/39e84130bbc9%2C35235%2C1731522975587.meta.1731522977959.meta, exclude list is [], retry=0 2024-11-13T18:36:17,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42373,DS-43627c6c-1ac2-4d78-b60b-b6c4203a554c,DISK] 2024-11-13T18:36:17,990 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/WALs/39e84130bbc9,35235,1731522975587/39e84130bbc9%2C35235%2C1731522975587.meta.1731522977959.meta 2024-11-13T18:36:17,991 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33713:33713)] 2024-11-13T18:36:17,991 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:36:17,994 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:36:17,998 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:36:18,003 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:36:18,009 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:36:18,009 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:18,010 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:36:18,010 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:36:18,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:36:18,015 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:36:18,016 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:18,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:18,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:36:18,019 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:36:18,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:18,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:18,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:36:18,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:36:18,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:18,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:18,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:36:18,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:36:18,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:18,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:36:18,026 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:36:18,028 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740 2024-11-13T18:36:18,031 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740 2024-11-13T18:36:18,034 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:36:18,034 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:36:18,035 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:36:18,038 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:36:18,039 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65991006, jitterRate=-0.016657382249832153}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:36:18,039 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:36:18,041 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731522978011Writing region info on filesystem at 1731522978011Initializing all the Stores at 1731522978013 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522978013Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522978014 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522978014Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522978014Cleaning up temporary data from old regions at 1731522978034 (+20 ms)Running coprocessor post-open hooks at 1731522978039 (+5 ms)Region opened successfully at 1731522978041 (+2 ms) 2024-11-13T18:36:18,049 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731522977925 2024-11-13T18:36:18,060 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:36:18,061 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:36:18,062 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:18,064 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39e84130bbc9,35235,1731522975587, state=OPEN 2024-11-13T18:36:18,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:36:18,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:36:18,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:36:18,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:36:18,070 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:18,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T18:36:18,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39e84130bbc9,35235,1731522975587 in 313 msec 2024-11-13T18:36:18,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T18:36:18,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 515 msec 2024-11-13T18:36:18,084 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:36:18,084 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T18:36:18,105 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:18,106 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:18,128 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:18,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43909, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:18,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1750 sec 2024-11-13T18:36:18,157 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731522978157, completionTime=-1 2024-11-13T18:36:18,160 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T18:36:18,161 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T18:36:18,192 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T18:36:18,193 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731523038192 2024-11-13T18:36:18,193 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731523098193 2024-11-13T18:36:18,193 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 32 msec 2024-11-13T18:36:18,196 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,36105,1731522974847-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:18,196 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,36105,1731522974847-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:18,196 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,36105,1731522974847-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:18,198 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-39e84130bbc9:36105, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:18,199 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:18,200 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:18,206 DEBUG [master/39e84130bbc9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T18:36:18,232 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.492sec 2024-11-13T18:36:18,233 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T18:36:18,235 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T18:36:18,236 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T18:36:18,236 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T18:36:18,236 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T18:36:18,237 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,36105,1731522974847-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:36:18,238 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,36105,1731522974847-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T18:36:18,246 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T18:36:18,247 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T18:36:18,248 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,36105,1731522974847-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:36:18,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:18,294 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T18:36:18,294 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T18:36:18,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:18,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:18,313 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4640898b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:18,325 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:18,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:18,329 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:18,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:18,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=39e84130bbc9,36105,1731522974847 2024-11-13T18:36:18,377 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=446, ProcessCount=11, AvailableMemoryMB=3065 2024-11-13T18:36:18,412 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:36:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 39e84130bbc9,36105,1731522974847 2024-11-13T18:36:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63ddab2d 2024-11-13T18:36:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:36:18,447 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:36:18,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:36:18,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:36:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:18,467 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:36:18,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T18:36:18,470 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:18,473 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:36:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:36:18,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741835_1011 (size=959) 2024-11-13T18:36:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:36:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:36:18,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 2024-11-13T18:36:18,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741836_1012 (size=53) 2024-11-13T18:36:18,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:18,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 430b0529634d320a32ba155ab3165681, disabling compactions & flushes 2024-11-13T18:36:18,908 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:18,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:18,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. after waiting 0 ms 2024-11-13T18:36:18,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:18,908 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:18,909 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 430b0529634d320a32ba155ab3165681: Waiting for close lock at 1731522978908Disabling compacts and flushes for region at 1731522978908Disabling writes for close at 1731522978908Writing region close event to WAL at 1731522978908Closed at 1731522978908 2024-11-13T18:36:18,911 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:36:18,916 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731522978911"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522978911"}]},"ts":"1731522978911"} 2024-11-13T18:36:18,922 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:36:18,923 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:36:18,927 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522978924"}]},"ts":"1731522978924"} 2024-11-13T18:36:18,932 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:36:18,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, ASSIGN}] 2024-11-13T18:36:18,937 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, ASSIGN 2024-11-13T18:36:18,938 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, ASSIGN; state=OFFLINE, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=false 2024-11-13T18:36:19,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=430b0529634d320a32ba155ab3165681, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:19,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, ASSIGN because future has completed 2024-11-13T18:36:19,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:36:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:36:19,259 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:19,259 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:36:19,260 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,260 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:19,260 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,260 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,263 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,266 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430b0529634d320a32ba155ab3165681 columnFamilyName A 2024-11-13T18:36:19,266 DEBUG [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:19,267 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] regionserver.HStore(327): Store=430b0529634d320a32ba155ab3165681/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:19,267 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,270 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430b0529634d320a32ba155ab3165681 columnFamilyName B 2024-11-13T18:36:19,270 DEBUG [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:19,271 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] regionserver.HStore(327): Store=430b0529634d320a32ba155ab3165681/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:19,271 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,273 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 430b0529634d320a32ba155ab3165681 columnFamilyName C 2024-11-13T18:36:19,274 DEBUG [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:19,274 INFO [StoreOpener-430b0529634d320a32ba155ab3165681-1 {}] regionserver.HStore(327): Store=430b0529634d320a32ba155ab3165681/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:19,275 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,276 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,277 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,279 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,280 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,281 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:36:19,283 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,290 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:36:19,291 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 430b0529634d320a32ba155ab3165681; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65116557, jitterRate=-0.029687687754631042}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:36:19,292 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,293 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 430b0529634d320a32ba155ab3165681: Running coprocessor pre-open hook at 1731522979260Writing region info on filesystem at 1731522979260Initializing all the Stores at 1731522979262 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522979262Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522979263 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522979263Cleaning up temporary data from old regions at 1731522979280 (+17 ms)Running coprocessor post-open hooks at 1731522979292 (+12 ms)Region opened successfully at 1731522979292 2024-11-13T18:36:19,295 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., pid=6, masterSystemTime=1731522979250 2024-11-13T18:36:19,299 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:19,299 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:19,300 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=430b0529634d320a32ba155ab3165681, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:19,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:36:19,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T18:36:19,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 in 210 msec 2024-11-13T18:36:19,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T18:36:19,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, ASSIGN in 377 msec 2024-11-13T18:36:19,317 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:36:19,318 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522979317"}]},"ts":"1731522979317"} 2024-11-13T18:36:19,321 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:36:19,323 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:36:19,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 866 msec 2024-11-13T18:36:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:36:19,615 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:19,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,625 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7747469c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,628 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f093bbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,630 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,636 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:36:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 39e84130bbc9,36105,1731522974847 2024-11-13T18:36:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68ee701a 2024-11-13T18:36:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:36:19,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:36:19,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,649 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7def95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,653 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c373c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,658 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@551748ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4834e559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,673 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ca9e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d05b08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,683 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35afed03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108777b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,690 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632c8962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,693 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f07addf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,696 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afb7308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8a593e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d04c0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,705 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7219b561, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,709 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159e5a17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,712 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1058791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:19,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:19,717 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b6d532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:19,719 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394100ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:19,726 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,726 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,726 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,726 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,733 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,738 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,739 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,754 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,764 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:19,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:19,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:19,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:19,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:36:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T18:36:19,780 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:36:19,783 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:19,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:19,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:19,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:36:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:36:19,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:19,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:19,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:19,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:19,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/9bed7856d84f4edabea8cc7672c63afc is 50, key is test_row_0/A:col10/1731522979839/Put/seqid=0 2024-11-13T18:36:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741837_1013 (size=12001) 2024-11-13T18:36:20,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/9bed7856d84f4edabea8cc7672c63afc 2024-11-13T18:36:20,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523040034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523040043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523040047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523040064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523040069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:36:20,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/699ba810500941108d6fc925601c26d0 is 50, key is test_row_0/B:col10/1731522979839/Put/seqid=0 2024-11-13T18:36:20,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:20,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523040186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523040186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523040187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523040188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523040188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741838_1014 (size=12001) 2024-11-13T18:36:20,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:36:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523040396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523040404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523040404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523040395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523040405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:20,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/699ba810500941108d6fc925601c26d0 2024-11-13T18:36:20,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/af28e8b2f4ab4e7ea0b17a54c139baa5 is 50, key is test_row_0/C:col10/1731522979839/Put/seqid=0 2024-11-13T18:36:20,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:20,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:20,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741839_1015 (size=12001) 2024-11-13T18:36:20,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523040714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523040715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523040724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523040725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523040725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:20,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:36:20,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:20,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/af28e8b2f4ab4e7ea0b17a54c139baa5 2024-11-13T18:36:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/9bed7856d84f4edabea8cc7672c63afc as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9bed7856d84f4edabea8cc7672c63afc 2024-11-13T18:36:21,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9bed7856d84f4edabea8cc7672c63afc, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:36:21,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/699ba810500941108d6fc925601c26d0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/699ba810500941108d6fc925601c26d0 2024-11-13T18:36:21,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/699ba810500941108d6fc925601c26d0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:36:21,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/af28e8b2f4ab4e7ea0b17a54c139baa5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/af28e8b2f4ab4e7ea0b17a54c139baa5 2024-11-13T18:36:21,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/af28e8b2f4ab4e7ea0b17a54c139baa5, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:36:21,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 430b0529634d320a32ba155ab3165681 in 1320ms, sequenceid=13, compaction requested=false 2024-11-13T18:36:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T18:36:21,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:21,245 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:21,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:36:21,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/41d3b1a80d024cada3807840e471a9b2 is 50, key is test_row_0/A:col10/1731522980034/Put/seqid=0 2024-11-13T18:36:21,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523041266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523041269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523041270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523041273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523041281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:21,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:21,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741840_1016 (size=14341) 2024-11-13T18:36:21,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/41d3b1a80d024cada3807840e471a9b2 2024-11-13T18:36:21,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/fa9734218de8459f82ada23061493d5f is 50, key is test_row_0/B:col10/1731522980034/Put/seqid=0 2024-11-13T18:36:21,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741841_1017 (size=12001) 2024-11-13T18:36:21,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/fa9734218de8459f82ada23061493d5f 2024-11-13T18:36:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523041384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523041384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523041394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523041394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/acc6e45ad5e8472d920b53a351155e21 is 50, key is test_row_0/C:col10/1731522980034/Put/seqid=0 2024-11-13T18:36:21,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523041404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741842_1018 (size=12001) 2024-11-13T18:36:21,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/acc6e45ad5e8472d920b53a351155e21 2024-11-13T18:36:21,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/41d3b1a80d024cada3807840e471a9b2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/41d3b1a80d024cada3807840e471a9b2 2024-11-13T18:36:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/41d3b1a80d024cada3807840e471a9b2, entries=200, sequenceid=39, filesize=14.0 K 2024-11-13T18:36:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/fa9734218de8459f82ada23061493d5f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa9734218de8459f82ada23061493d5f 2024-11-13T18:36:21,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa9734218de8459f82ada23061493d5f, entries=150, sequenceid=39, filesize=11.7 K 2024-11-13T18:36:21,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/acc6e45ad5e8472d920b53a351155e21 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/acc6e45ad5e8472d920b53a351155e21 2024-11-13T18:36:21,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/acc6e45ad5e8472d920b53a351155e21, entries=150, sequenceid=39, filesize=11.7 K 2024-11-13T18:36:21,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 430b0529634d320a32ba155ab3165681 in 288ms, sequenceid=39, compaction requested=false 2024-11-13T18:36:21,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:21,608 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:21,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:36:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:21,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/95973431cd5645f58317be9448eb53f3 is 50, key is test_row_0/A:col10/1731522981602/Put/seqid=0 2024-11-13T18:36:21,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741843_1019 (size=11997) 2024-11-13T18:36:21,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/95973431cd5645f58317be9448eb53f3 2024-11-13T18:36:21,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/71857942fb5f483393f5f36bdd8619e2 is 50, key is test_row_0/B:col10/1731522981602/Put/seqid=0 2024-11-13T18:36:21,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741844_1020 (size=9657) 2024-11-13T18:36:21,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/71857942fb5f483393f5f36bdd8619e2 2024-11-13T18:36:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523041727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523041725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523041749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523041749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523041750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ebb19d191f6a4230a260f9c88e1b550c is 50, key is test_row_0/C:col10/1731522981602/Put/seqid=0 2024-11-13T18:36:21,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741845_1021 (size=9657) 2024-11-13T18:36:21,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ebb19d191f6a4230a260f9c88e1b550c 2024-11-13T18:36:21,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523041854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523041855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523041875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523041876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523041883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:21,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/95973431cd5645f58317be9448eb53f3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95973431cd5645f58317be9448eb53f3 2024-11-13T18:36:21,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95973431cd5645f58317be9448eb53f3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T18:36:21,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/71857942fb5f483393f5f36bdd8619e2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/71857942fb5f483393f5f36bdd8619e2 2024-11-13T18:36:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:36:21,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:21,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:21,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:21,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:21,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/71857942fb5f483393f5f36bdd8619e2, entries=100, sequenceid=50, filesize=9.4 K 2024-11-13T18:36:21,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ebb19d191f6a4230a260f9c88e1b550c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ebb19d191f6a4230a260f9c88e1b550c 2024-11-13T18:36:22,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ebb19d191f6a4230a260f9c88e1b550c, entries=100, sequenceid=50, filesize=9.4 K 2024-11-13T18:36:22,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 430b0529634d320a32ba155ab3165681 in 398ms, sequenceid=50, compaction requested=true 2024-11-13T18:36:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:22,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:22,035 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:22,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:22,037 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:22,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:22,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:22,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:22,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:22,040 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:22,040 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:22,042 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/B is initiating minor compaction (all files) 2024-11-13T18:36:22,043 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/B in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,043 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/699ba810500941108d6fc925601c26d0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa9734218de8459f82ada23061493d5f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/71857942fb5f483393f5f36bdd8619e2] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=32.9 K 2024-11-13T18:36:22,044 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/A is initiating minor compaction (all files) 2024-11-13T18:36:22,044 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/A in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,044 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9bed7856d84f4edabea8cc7672c63afc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/41d3b1a80d024cada3807840e471a9b2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95973431cd5645f58317be9448eb53f3] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=37.4 K 2024-11-13T18:36:22,045 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 699ba810500941108d6fc925601c26d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522979824 2024-11-13T18:36:22,046 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bed7856d84f4edabea8cc7672c63afc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522979824 2024-11-13T18:36:22,046 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9734218de8459f82ada23061493d5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731522980034 2024-11-13T18:36:22,047 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d3b1a80d024cada3807840e471a9b2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731522980034 2024-11-13T18:36:22,048 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 71857942fb5f483393f5f36bdd8619e2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731522981269 2024-11-13T18:36:22,048 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95973431cd5645f58317be9448eb53f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731522981269 2024-11-13T18:36:22,092 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#B#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:22,093 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/463914939a2c436581149a3ef5c00bf1 is 50, key is test_row_0/B:col10/1731522981602/Put/seqid=0 2024-11-13T18:36:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:22,100 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:22,100 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#A#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:22,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-13T18:36:22,101 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/835ac9e5b6ff41d99d78e44d75261c2f is 50, key is test_row_0/A:col10/1731522981602/Put/seqid=0 2024-11-13T18:36:22,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741846_1022 (size=12104) 2024-11-13T18:36:22,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523042125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523042130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523042133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741847_1023 (size=12104) 2024-11-13T18:36:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523042137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523042135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/feb2f77388ca4e19be98b580505bde27 is 50, key is test_row_0/A:col10/1731522981711/Put/seqid=0 2024-11-13T18:36:22,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741848_1024 (size=12001) 2024-11-13T18:36:22,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523042254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523042254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523042263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523042264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523042264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:22,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:22,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:22,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523042464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523042474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523042474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523042487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523042487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,562 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/463914939a2c436581149a3ef5c00bf1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/463914939a2c436581149a3ef5c00bf1 2024-11-13T18:36:22,571 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/835ac9e5b6ff41d99d78e44d75261c2f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/835ac9e5b6ff41d99d78e44d75261c2f 2024-11-13T18:36:22,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,593 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/A of 430b0529634d320a32ba155ab3165681 into 835ac9e5b6ff41d99d78e44d75261c2f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:22,594 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:22,595 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/B of 430b0529634d320a32ba155ab3165681 into 463914939a2c436581149a3ef5c00bf1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:22,596 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:22,598 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/B, priority=13, startTime=1731522982037; duration=0sec 2024-11-13T18:36:22,598 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/A, priority=13, startTime=1731522982009; duration=0sec 2024-11-13T18:36:22,598 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:22,598 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:22,598 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:22,598 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:22,599 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:22,604 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:22,604 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/C is initiating minor compaction (all files) 2024-11-13T18:36:22,604 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/C in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,605 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/af28e8b2f4ab4e7ea0b17a54c139baa5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/acc6e45ad5e8472d920b53a351155e21, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ebb19d191f6a4230a260f9c88e1b550c] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=32.9 K 2024-11-13T18:36:22,607 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting af28e8b2f4ab4e7ea0b17a54c139baa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731522979824 2024-11-13T18:36:22,608 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting acc6e45ad5e8472d920b53a351155e21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731522980034 2024-11-13T18:36:22,609 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb19d191f6a4230a260f9c88e1b550c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731522981269 2024-11-13T18:36:22,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/feb2f77388ca4e19be98b580505bde27 2024-11-13T18:36:22,642 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:22,643 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/b4bd411c1cbe4592a32dcf247438f111 is 50, key is test_row_0/C:col10/1731522981602/Put/seqid=0 2024-11-13T18:36:22,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/fa4b00398fc042fc878b63cd949c4ac9 is 50, key is test_row_0/B:col10/1731522981711/Put/seqid=0 2024-11-13T18:36:22,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741849_1025 (size=12104) 2024-11-13T18:36:22,718 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/b4bd411c1cbe4592a32dcf247438f111 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b4bd411c1cbe4592a32dcf247438f111 2024-11-13T18:36:22,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741850_1026 (size=12001) 2024-11-13T18:36:22,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/fa4b00398fc042fc878b63cd949c4ac9 2024-11-13T18:36:22,731 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/C of 430b0529634d320a32ba155ab3165681 into b4bd411c1cbe4592a32dcf247438f111(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:22,732 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:22,732 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/C, priority=13, startTime=1731522982037; duration=0sec 2024-11-13T18:36:22,732 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:22,732 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:22,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:22,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/1d020c0b0f4a4a6689d19d495d4ffe63 is 50, key is test_row_0/C:col10/1731522981711/Put/seqid=0 2024-11-13T18:36:22,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523042774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741851_1027 (size=12001) 2024-11-13T18:36:22,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/1d020c0b0f4a4a6689d19d495d4ffe63 2024-11-13T18:36:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523042784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/feb2f77388ca4e19be98b580505bde27 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/feb2f77388ca4e19be98b580505bde27 2024-11-13T18:36:22,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523042794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523042803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523042804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:22,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/feb2f77388ca4e19be98b580505bde27, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:36:22,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/fa4b00398fc042fc878b63cd949c4ac9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa4b00398fc042fc878b63cd949c4ac9 2024-11-13T18:36:22,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa4b00398fc042fc878b63cd949c4ac9, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:36:22,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/1d020c0b0f4a4a6689d19d495d4ffe63 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1d020c0b0f4a4a6689d19d495d4ffe63 2024-11-13T18:36:22,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1d020c0b0f4a4a6689d19d495d4ffe63, entries=150, sequenceid=80, filesize=11.7 K 2024-11-13T18:36:22,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=33.54 KB/34350 for 430b0529634d320a32ba155ab3165681 in 746ms, sequenceid=80, compaction requested=false 2024-11-13T18:36:22,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:22,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T18:36:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:22,905 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T18:36:22,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/ddd9e496bc3f4adbacf5bdd96bd68353 is 50, key is test_row_0/A:col10/1731522982120/Put/seqid=0 2024-11-13T18:36:22,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741852_1028 (size=12001) 2024-11-13T18:36:22,951 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/ddd9e496bc3f4adbacf5bdd96bd68353 2024-11-13T18:36:22,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/48e808bf306f498c85e2f204f661fda6 is 50, key is test_row_0/B:col10/1731522982120/Put/seqid=0 2024-11-13T18:36:23,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741853_1029 (size=12001) 2024-11-13T18:36:23,030 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/48e808bf306f498c85e2f204f661fda6 2024-11-13T18:36:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/046ce17ba1f24089ba8c5b5d755671f6 is 50, key is test_row_0/C:col10/1731522982120/Put/seqid=0 2024-11-13T18:36:23,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741854_1030 (size=12001) 2024-11-13T18:36:23,083 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/046ce17ba1f24089ba8c5b5d755671f6 2024-11-13T18:36:23,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/ddd9e496bc3f4adbacf5bdd96bd68353 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ddd9e496bc3f4adbacf5bdd96bd68353 2024-11-13T18:36:23,110 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ddd9e496bc3f4adbacf5bdd96bd68353, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T18:36:23,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/48e808bf306f498c85e2f204f661fda6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/48e808bf306f498c85e2f204f661fda6 2024-11-13T18:36:23,124 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/48e808bf306f498c85e2f204f661fda6, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T18:36:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/046ce17ba1f24089ba8c5b5d755671f6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/046ce17ba1f24089ba8c5b5d755671f6 2024-11-13T18:36:23,137 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/046ce17ba1f24089ba8c5b5d755671f6, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T18:36:23,140 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 430b0529634d320a32ba155ab3165681 in 235ms, sequenceid=91, compaction requested=true 2024-11-13T18:36:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T18:36:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T18:36:23,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T18:36:23,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3690 sec 2024-11-13T18:36:23,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.3910 sec 2024-11-13T18:36:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:23,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:36:23,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/95972265564645eaa5a797aed5eed7f3 is 50, key is test_row_0/A:col10/1731522983309/Put/seqid=0 2024-11-13T18:36:23,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741855_1031 (size=12001) 2024-11-13T18:36:23,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/95972265564645eaa5a797aed5eed7f3 2024-11-13T18:36:23,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/285fa4c23a4d47a59136f6f7974dc55c is 50, key is test_row_0/B:col10/1731522983309/Put/seqid=0 2024-11-13T18:36:23,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523043424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523043435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523043438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523043441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523043442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741856_1032 (size=12001) 2024-11-13T18:36:23,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/285fa4c23a4d47a59136f6f7974dc55c 2024-11-13T18:36:23,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/88a80045c176482ba7fa3e868bec0b06 is 50, key is test_row_0/C:col10/1731522983309/Put/seqid=0 2024-11-13T18:36:23,531 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:36:23,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741857_1033 (size=12001) 2024-11-13T18:36:23,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523043555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523043561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523043569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523043569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523043569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,638 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T18:36:23,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523043776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523043784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523043785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523043785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523043785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:23,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/88a80045c176482ba7fa3e868bec0b06 2024-11-13T18:36:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T18:36:23,954 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:23,955 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T18:36:23,961 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:36:23,963 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:23,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:23,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/95972265564645eaa5a797aed5eed7f3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95972265564645eaa5a797aed5eed7f3 2024-11-13T18:36:23,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95972265564645eaa5a797aed5eed7f3, entries=150, sequenceid=103, filesize=11.7 K 2024-11-13T18:36:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/285fa4c23a4d47a59136f6f7974dc55c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/285fa4c23a4d47a59136f6f7974dc55c 2024-11-13T18:36:24,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/285fa4c23a4d47a59136f6f7974dc55c, entries=150, sequenceid=103, filesize=11.7 K 2024-11-13T18:36:24,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/88a80045c176482ba7fa3e868bec0b06 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/88a80045c176482ba7fa3e868bec0b06 2024-11-13T18:36:24,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/88a80045c176482ba7fa3e868bec0b06, entries=150, sequenceid=103, filesize=11.7 K 2024-11-13T18:36:24,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 430b0529634d320a32ba155ab3165681 in 720ms, sequenceid=103, compaction requested=true 2024-11-13T18:36:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:24,053 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:36:24,054 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:24,057 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:24,057 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:24,057 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/B is initiating minor compaction (all files) 2024-11-13T18:36:24,057 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/A is initiating minor compaction (all files) 2024-11-13T18:36:24,058 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/B in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:24,058 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/A in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:24,058 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/463914939a2c436581149a3ef5c00bf1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa4b00398fc042fc878b63cd949c4ac9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/48e808bf306f498c85e2f204f661fda6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/285fa4c23a4d47a59136f6f7974dc55c] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=47.0 K 2024-11-13T18:36:24,058 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/835ac9e5b6ff41d99d78e44d75261c2f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/feb2f77388ca4e19be98b580505bde27, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ddd9e496bc3f4adbacf5bdd96bd68353, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95972265564645eaa5a797aed5eed7f3] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=47.0 K 2024-11-13T18:36:24,059 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 463914939a2c436581149a3ef5c00bf1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731522980054 2024-11-13T18:36:24,060 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4b00398fc042fc878b63cd949c4ac9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731522981711 2024-11-13T18:36:24,060 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835ac9e5b6ff41d99d78e44d75261c2f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731522980054 2024-11-13T18:36:24,060 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 48e808bf306f498c85e2f204f661fda6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731522982120 2024-11-13T18:36:24,062 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 285fa4c23a4d47a59136f6f7974dc55c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731522983309 2024-11-13T18:36:24,062 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting feb2f77388ca4e19be98b580505bde27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731522981711 2024-11-13T18:36:24,063 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddd9e496bc3f4adbacf5bdd96bd68353, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731522982120 2024-11-13T18:36:24,065 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95972265564645eaa5a797aed5eed7f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731522983309 2024-11-13T18:36:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:36:24,093 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#B#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:24,094 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/55535ec929bb43a8819a4fafaaabb258 is 50, key is test_row_0/B:col10/1731522983309/Put/seqid=0 2024-11-13T18:36:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:24,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:36:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:24,104 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:24,106 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/ed4f08e0d5d14022805b303891100bd0 is 50, key is test_row_0/A:col10/1731522983309/Put/seqid=0 2024-11-13T18:36:24,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/71a1dccc655a4e04b5372256e7cf4d4d is 50, key is test_row_0/A:col10/1731522983437/Put/seqid=0 2024-11-13T18:36:24,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:36:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:24,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:24,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:24,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741858_1034 (size=12241) 2024-11-13T18:36:24,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,158 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/55535ec929bb43a8819a4fafaaabb258 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/55535ec929bb43a8819a4fafaaabb258 2024-11-13T18:36:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523044141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523044149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523044151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523044154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523044155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741859_1035 (size=12241) 2024-11-13T18:36:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,194 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/B of 430b0529634d320a32ba155ab3165681 into 55535ec929bb43a8819a4fafaaabb258(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:24,194 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:24,194 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/B, priority=12, startTime=1731522984053; duration=0sec 2024-11-13T18:36:24,194 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:24,194 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:24,195 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:24,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741860_1036 (size=14341) 2024-11-13T18:36:24,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/71a1dccc655a4e04b5372256e7cf4d4d 2024-11-13T18:36:24,202 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:24,202 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/C is initiating minor compaction (all files) 2024-11-13T18:36:24,202 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/C in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:24,203 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b4bd411c1cbe4592a32dcf247438f111, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1d020c0b0f4a4a6689d19d495d4ffe63, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/046ce17ba1f24089ba8c5b5d755671f6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/88a80045c176482ba7fa3e868bec0b06] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=47.0 K 2024-11-13T18:36:24,205 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b4bd411c1cbe4592a32dcf247438f111, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731522980054 2024-11-13T18:36:24,207 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d020c0b0f4a4a6689d19d495d4ffe63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731522981711 2024-11-13T18:36:24,207 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/ed4f08e0d5d14022805b303891100bd0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ed4f08e0d5d14022805b303891100bd0 2024-11-13T18:36:24,210 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 046ce17ba1f24089ba8c5b5d755671f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731522982120 2024-11-13T18:36:24,212 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a80045c176482ba7fa3e868bec0b06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731522983309 2024-11-13T18:36:24,228 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/A of 430b0529634d320a32ba155ab3165681 into ed4f08e0d5d14022805b303891100bd0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:24,228 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:24,228 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/A, priority=12, startTime=1731522984052; duration=0sec 2024-11-13T18:36:24,229 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:24,229 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:24,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/947095edbca0457c9eef01b2e45f6059 is 50, key is test_row_0/B:col10/1731522983437/Put/seqid=0 2024-11-13T18:36:24,245 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#C#compaction#25 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:24,248 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/a0d1d959748047e4bedc104976c818b0 is 50, key is test_row_0/C:col10/1731522983309/Put/seqid=0 2024-11-13T18:36:24,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741861_1037 (size=12001) 2024-11-13T18:36:24,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/947095edbca0457c9eef01b2e45f6059 2024-11-13T18:36:24,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:36:24,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:24,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:24,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:24,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:24,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:36:24,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523044274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523044274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523044274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523044275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523044283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741862_1038 (size=12241) 2024-11-13T18:36:24,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/4234b0d919f04779870ff50d7b290b7b is 50, key is test_row_0/C:col10/1731522983437/Put/seqid=0 2024-11-13T18:36:24,314 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/a0d1d959748047e4bedc104976c818b0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a0d1d959748047e4bedc104976c818b0 2024-11-13T18:36:24,360 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/C of 430b0529634d320a32ba155ab3165681 into a0d1d959748047e4bedc104976c818b0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:24,360 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:24,360 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/C, priority=12, startTime=1731522984053; duration=0sec 2024-11-13T18:36:24,360 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:24,361 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:24,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741863_1039 (size=12001) 2024-11-13T18:36:24,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/4234b0d919f04779870ff50d7b290b7b 2024-11-13T18:36:24,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/71a1dccc655a4e04b5372256e7cf4d4d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/71a1dccc655a4e04b5372256e7cf4d4d 2024-11-13T18:36:24,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/71a1dccc655a4e04b5372256e7cf4d4d, entries=200, sequenceid=128, filesize=14.0 K 2024-11-13T18:36:24,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/947095edbca0457c9eef01b2e45f6059 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/947095edbca0457c9eef01b2e45f6059 2024-11-13T18:36:24,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/947095edbca0457c9eef01b2e45f6059, entries=150, sequenceid=128, filesize=11.7 K 2024-11-13T18:36:24,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/4234b0d919f04779870ff50d7b290b7b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/4234b0d919f04779870ff50d7b290b7b 2024-11-13T18:36:24,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/4234b0d919f04779870ff50d7b290b7b, entries=150, sequenceid=128, filesize=11.7 K 2024-11-13T18:36:24,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 430b0529634d320a32ba155ab3165681 in 324ms, sequenceid=128, compaction requested=false 2024-11-13T18:36:24,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:24,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T18:36:24,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:24,434 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:36:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/6d0b0e5e574540e99c9508237d48aaa1 is 50, key is test_row_0/A:col10/1731522984143/Put/seqid=0 2024-11-13T18:36:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:24,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:24,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741864_1040 (size=12151) 2024-11-13T18:36:24,506 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/6d0b0e5e574540e99c9508237d48aaa1 2024-11-13T18:36:24,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/a2c1462d03a042c781f5d0bfb4270bbe is 50, key is test_row_0/B:col10/1731522984143/Put/seqid=0 2024-11-13T18:36:24,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741865_1041 (size=12151) 2024-11-13T18:36:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523044558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523044563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523044561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523044571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523044571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:36:24,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523044684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523044684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523044684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523044684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523044685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523044894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523044894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523044903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523044904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523044904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:24,954 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/a2c1462d03a042c781f5d0bfb4270bbe 2024-11-13T18:36:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/b39c25489061412c9e75413f73e96c88 is 50, key is test_row_0/C:col10/1731522984143/Put/seqid=0 2024-11-13T18:36:25,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741866_1042 (size=12151) 2024-11-13T18:36:25,068 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/b39c25489061412c9e75413f73e96c88 2024-11-13T18:36:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/6d0b0e5e574540e99c9508237d48aaa1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/6d0b0e5e574540e99c9508237d48aaa1 2024-11-13T18:36:25,093 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/6d0b0e5e574540e99c9508237d48aaa1, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T18:36:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/a2c1462d03a042c781f5d0bfb4270bbe as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/a2c1462d03a042c781f5d0bfb4270bbe 2024-11-13T18:36:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:36:25,123 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/a2c1462d03a042c781f5d0bfb4270bbe, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T18:36:25,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/b39c25489061412c9e75413f73e96c88 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b39c25489061412c9e75413f73e96c88 2024-11-13T18:36:25,138 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b39c25489061412c9e75413f73e96c88, entries=150, sequenceid=142, filesize=11.9 K 2024-11-13T18:36:25,143 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 430b0529634d320a32ba155ab3165681 in 709ms, sequenceid=142, compaction requested=true 2024-11-13T18:36:25,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:25,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:25,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T18:36:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T18:36:25,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T18:36:25,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1840 sec 2024-11-13T18:36:25,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2030 sec 2024-11-13T18:36:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:25,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:25,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:36:25,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/b9ef998030224b70bd708f1fc1f683b8 is 50, key is test_row_0/A:col10/1731522985217/Put/seqid=0 2024-11-13T18:36:25,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523045243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523045246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523045247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523045250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523045253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741867_1043 (size=12151) 2024-11-13T18:36:25,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/b9ef998030224b70bd708f1fc1f683b8 2024-11-13T18:36:25,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T18:36:25,294 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T18:36:25,296 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:36:25,296 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T18:36:25,297 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T18:36:25,297 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T18:36:25,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:36:25,298 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T18:36:25,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/0bb983c92fac4742bb4431ce671c60d3 is 50, key is test_row_0/B:col10/1731522985217/Put/seqid=0 2024-11-13T18:36:25,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741868_1044 (size=12151) 2024-11-13T18:36:25,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/0bb983c92fac4742bb4431ce671c60d3 2024-11-13T18:36:25,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/53afe89d0115406cbe37a62207e96a88 is 50, key is test_row_0/C:col10/1731522985217/Put/seqid=0 2024-11-13T18:36:25,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523045366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523045366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523045367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523045367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523045366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741869_1045 (size=12151) 2024-11-13T18:36:25,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523045583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523045584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523045584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523045584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523045584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:25,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:25,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:25,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/53afe89d0115406cbe37a62207e96a88 2024-11-13T18:36:25,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/b9ef998030224b70bd708f1fc1f683b8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b9ef998030224b70bd708f1fc1f683b8 2024-11-13T18:36:25,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b9ef998030224b70bd708f1fc1f683b8, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T18:36:25,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/0bb983c92fac4742bb4431ce671c60d3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0bb983c92fac4742bb4431ce671c60d3 2024-11-13T18:36:25,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0bb983c92fac4742bb4431ce671c60d3, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T18:36:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/53afe89d0115406cbe37a62207e96a88 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/53afe89d0115406cbe37a62207e96a88 2024-11-13T18:36:25,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/53afe89d0115406cbe37a62207e96a88, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T18:36:25,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 430b0529634d320a32ba155ab3165681 in 644ms, sequenceid=170, compaction requested=true 2024-11-13T18:36:25,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:25,866 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:25,866 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:25,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:25,869 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50884 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:25,869 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/A is initiating minor compaction (all files) 2024-11-13T18:36:25,869 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/A in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:25,869 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ed4f08e0d5d14022805b303891100bd0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/71a1dccc655a4e04b5372256e7cf4d4d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/6d0b0e5e574540e99c9508237d48aaa1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b9ef998030224b70bd708f1fc1f683b8] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=49.7 K 2024-11-13T18:36:25,869 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:25,870 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/B is initiating minor compaction (all files) 2024-11-13T18:36:25,870 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/B in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:25,870 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/55535ec929bb43a8819a4fafaaabb258, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/947095edbca0457c9eef01b2e45f6059, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/a2c1462d03a042c781f5d0bfb4270bbe, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0bb983c92fac4742bb4431ce671c60d3] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=47.4 K 2024-11-13T18:36:25,871 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55535ec929bb43a8819a4fafaaabb258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731522983309 2024-11-13T18:36:25,872 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ed4f08e0d5d14022805b303891100bd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731522983309 2024-11-13T18:36:25,872 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 947095edbca0457c9eef01b2e45f6059, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522983437 2024-11-13T18:36:25,872 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a1dccc655a4e04b5372256e7cf4d4d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522983423 2024-11-13T18:36:25,873 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c1462d03a042c781f5d0bfb4270bbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731522984131 2024-11-13T18:36:25,873 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0b0e5e574540e99c9508237d48aaa1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731522984131 2024-11-13T18:36:25,876 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ef998030224b70bd708f1fc1f683b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731522985217 2024-11-13T18:36:25,877 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bb983c92fac4742bb4431ce671c60d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731522985217 2024-11-13T18:36:25,902 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#A#compaction#33 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:25,903 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/7453ed8f6ad641548eff07f4b0c68916 is 50, key is test_row_0/A:col10/1731522985217/Put/seqid=0 2024-11-13T18:36:25,905 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:25,905 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/1d08f51bdbe94fdb93f31e3618bbfd4a is 50, key is test_row_0/B:col10/1731522985217/Put/seqid=0 2024-11-13T18:36:25,911 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:25,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:36:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:25,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/a51859e9997e4edcbab81ea8afd1b0f2 is 50, key is test_row_0/A:col10/1731522985907/Put/seqid=0 2024-11-13T18:36:25,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741870_1046 (size=12527) 2024-11-13T18:36:25,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741871_1047 (size=12527) 2024-11-13T18:36:25,993 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/1d08f51bdbe94fdb93f31e3618bbfd4a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/1d08f51bdbe94fdb93f31e3618bbfd4a 2024-11-13T18:36:26,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741872_1048 (size=14541) 2024-11-13T18:36:26,005 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/B of 430b0529634d320a32ba155ab3165681 into 1d08f51bdbe94fdb93f31e3618bbfd4a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:26,005 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:26,005 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/B, priority=12, startTime=1731522985866; duration=0sec 2024-11-13T18:36:26,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/a51859e9997e4edcbab81ea8afd1b0f2 2024-11-13T18:36:26,006 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:26,006 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:26,006 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:26,009 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:26,010 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/C is initiating minor compaction (all files) 2024-11-13T18:36:26,010 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/C in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,010 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a0d1d959748047e4bedc104976c818b0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/4234b0d919f04779870ff50d7b290b7b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b39c25489061412c9e75413f73e96c88, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/53afe89d0115406cbe37a62207e96a88] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=47.4 K 2024-11-13T18:36:26,011 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d1d959748047e4bedc104976c818b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731522983309 2024-11-13T18:36:26,012 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4234b0d919f04779870ff50d7b290b7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731522983437 2024-11-13T18:36:26,013 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b39c25489061412c9e75413f73e96c88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731522984131 2024-11-13T18:36:26,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,014 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53afe89d0115406cbe37a62207e96a88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731522985217 2024-11-13T18:36:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523045989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523045993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523045995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523045994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523045997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/440650511d5c4d6992940a6bb7e22525 is 50, key is test_row_0/B:col10/1731522985907/Put/seqid=0 2024-11-13T18:36:26,049 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#C#compaction#37 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:26,050 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/c158e7f8465a4f149464c359ab51b929 is 50, key is test_row_0/C:col10/1731522985217/Put/seqid=0 2024-11-13T18:36:26,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741873_1049 (size=12151) 2024-11-13T18:36:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T18:36:26,118 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:26,119 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T18:36:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:36:26,123 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:26,124 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:26,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:26,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523046126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523046126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523046137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741874_1050 (size=12527) 2024-11-13T18:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523046137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523046138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,164 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/c158e7f8465a4f149464c359ab51b929 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c158e7f8465a4f149464c359ab51b929 2024-11-13T18:36:26,179 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/C of 430b0529634d320a32ba155ab3165681 into c158e7f8465a4f149464c359ab51b929(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:26,179 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:26,180 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/C, priority=12, startTime=1731522985867; duration=0sec 2024-11-13T18:36:26,180 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:26,180 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:36:26,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:26,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:26,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523046343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523046344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523046363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523046366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523046365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,377 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/7453ed8f6ad641548eff07f4b0c68916 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7453ed8f6ad641548eff07f4b0c68916 2024-11-13T18:36:26,395 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/A of 430b0529634d320a32ba155ab3165681 into 7453ed8f6ad641548eff07f4b0c68916(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:26,395 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:26,395 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/A, priority=12, startTime=1731522985866; duration=0sec 2024-11-13T18:36:26,395 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:26,395 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:26,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:36:26,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/440650511d5c4d6992940a6bb7e22525 2024-11-13T18:36:26,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/94707c36f2ee4e09ab333037e9b5686c is 50, key is test_row_0/C:col10/1731522985907/Put/seqid=0 2024-11-13T18:36:26,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741875_1051 (size=12151) 2024-11-13T18:36:26,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/94707c36f2ee4e09ab333037e9b5686c 2024-11-13T18:36:26,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/a51859e9997e4edcbab81ea8afd1b0f2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a51859e9997e4edcbab81ea8afd1b0f2 2024-11-13T18:36:26,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a51859e9997e4edcbab81ea8afd1b0f2, entries=200, sequenceid=181, filesize=14.2 K 2024-11-13T18:36:26,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/440650511d5c4d6992940a6bb7e22525 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/440650511d5c4d6992940a6bb7e22525 2024-11-13T18:36:26,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/440650511d5c4d6992940a6bb7e22525, entries=150, sequenceid=181, filesize=11.9 K 2024-11-13T18:36:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/94707c36f2ee4e09ab333037e9b5686c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/94707c36f2ee4e09ab333037e9b5686c 2024-11-13T18:36:26,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523046654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/94707c36f2ee4e09ab333037e9b5686c, entries=150, sequenceid=181, filesize=11.9 K 2024-11-13T18:36:26,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 430b0529634d320a32ba155ab3165681 in 775ms, sequenceid=181, compaction requested=false 2024-11-13T18:36:26,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:26,688 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:26,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-13T18:36:26,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/f24c2003026543ecb4d00795446aaee5 is 50, key is test_row_0/A:col10/1731522986686/Put/seqid=0 2024-11-13T18:36:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523046710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523046712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523046713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523046718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741876_1052 (size=12151) 2024-11-13T18:36:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:36:26,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:26,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:26,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523046824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523046825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523046826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523046833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:26,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:26,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:26,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:26,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523047044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523047047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523047048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523047054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/f24c2003026543ecb4d00795446aaee5 2024-11-13T18:36:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523047164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/debdff204a78404f96d2b158e5217e6b is 50, key is test_row_0/B:col10/1731522986686/Put/seqid=0 2024-11-13T18:36:27,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741877_1053 (size=12151) 2024-11-13T18:36:27,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:27,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:27,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:36:27,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523047355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523047364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523047364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523047374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:27,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:27,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:27,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:27,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/debdff204a78404f96d2b158e5217e6b 2024-11-13T18:36:27,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/689a000d2cbe46e8b2bd8d815475a8b2 is 50, key is test_row_0/C:col10/1731522986686/Put/seqid=0 2024-11-13T18:36:27,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741878_1054 (size=12151) 2024-11-13T18:36:27,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/689a000d2cbe46e8b2bd8d815475a8b2 2024-11-13T18:36:27,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/f24c2003026543ecb4d00795446aaee5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/f24c2003026543ecb4d00795446aaee5 2024-11-13T18:36:27,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/f24c2003026543ecb4d00795446aaee5, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T18:36:27,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/debdff204a78404f96d2b158e5217e6b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/debdff204a78404f96d2b158e5217e6b 2024-11-13T18:36:27,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/debdff204a78404f96d2b158e5217e6b, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T18:36:27,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/689a000d2cbe46e8b2bd8d815475a8b2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/689a000d2cbe46e8b2bd8d815475a8b2 2024-11-13T18:36:27,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/689a000d2cbe46e8b2bd8d815475a8b2, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T18:36:27,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 430b0529634d320a32ba155ab3165681 in 1042ms, sequenceid=213, compaction requested=true 2024-11-13T18:36:27,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:27,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:27,731 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:27,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:27,732 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:27,733 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:27,734 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/A is initiating minor compaction (all files) 2024-11-13T18:36:27,734 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/A in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,734 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7453ed8f6ad641548eff07f4b0c68916, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a51859e9997e4edcbab81ea8afd1b0f2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/f24c2003026543ecb4d00795446aaee5] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=38.3 K 2024-11-13T18:36:27,735 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7453ed8f6ad641548eff07f4b0c68916, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731522985217 2024-11-13T18:36:27,736 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:27,736 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/B is initiating minor compaction (all files) 2024-11-13T18:36:27,736 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/B in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,737 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/1d08f51bdbe94fdb93f31e3618bbfd4a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/440650511d5c4d6992940a6bb7e22525, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/debdff204a78404f96d2b158e5217e6b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.0 K 2024-11-13T18:36:27,737 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a51859e9997e4edcbab81ea8afd1b0f2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731522985246 2024-11-13T18:36:27,737 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d08f51bdbe94fdb93f31e3618bbfd4a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731522985217 2024-11-13T18:36:27,738 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f24c2003026543ecb4d00795446aaee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731522986665 2024-11-13T18:36:27,738 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 440650511d5c4d6992940a6bb7e22525, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731522985246 2024-11-13T18:36:27,740 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting debdff204a78404f96d2b158e5217e6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731522986665 2024-11-13T18:36:27,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T18:36:27,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,742 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T18:36:27,759 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:27,760 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/b54efc860ab944cbb9d4e9b41ec1d13f is 50, key is test_row_0/A:col10/1731522986686/Put/seqid=0 2024-11-13T18:36:27,768 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:27,769 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/0e5eca012bd0473e97a7bc56e3f4950f is 50, key is test_row_0/B:col10/1731522986686/Put/seqid=0 2024-11-13T18:36:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/baa707b2ae854dfa89574d5f1bd7104f is 50, key is test_row_0/A:col10/1731522986709/Put/seqid=0 2024-11-13T18:36:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741879_1055 (size=12629) 2024-11-13T18:36:27,867 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/b54efc860ab944cbb9d4e9b41ec1d13f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b54efc860ab944cbb9d4e9b41ec1d13f 2024-11-13T18:36:27,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741880_1056 (size=12629) 2024-11-13T18:36:27,881 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/A of 430b0529634d320a32ba155ab3165681 into b54efc860ab944cbb9d4e9b41ec1d13f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:27,881 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:27,882 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/A, priority=13, startTime=1731522987731; duration=0sec 2024-11-13T18:36:27,882 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:27,882 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:27,882 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:27,884 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:27,884 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/C is initiating minor compaction (all files) 2024-11-13T18:36:27,884 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/C in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:27,884 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c158e7f8465a4f149464c359ab51b929, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/94707c36f2ee4e09ab333037e9b5686c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/689a000d2cbe46e8b2bd8d815475a8b2] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.0 K 2024-11-13T18:36:27,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:27,885 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c158e7f8465a4f149464c359ab51b929, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731522985217 2024-11-13T18:36:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:27,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741881_1057 (size=12151) 2024-11-13T18:36:27,888 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94707c36f2ee4e09ab333037e9b5686c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731522985246 2024-11-13T18:36:27,888 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/baa707b2ae854dfa89574d5f1bd7104f 2024-11-13T18:36:27,889 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 689a000d2cbe46e8b2bd8d815475a8b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731522986665 2024-11-13T18:36:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/ebf47cd69a324b28ac0c3342f5b3ebea is 50, key is test_row_0/B:col10/1731522986709/Put/seqid=0 2024-11-13T18:36:27,926 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#C#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:27,927 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/581735a1f1f54f96aef018c8f3173860 is 50, key is test_row_0/C:col10/1731522986686/Put/seqid=0 2024-11-13T18:36:27,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741882_1058 (size=12151) 2024-11-13T18:36:27,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741883_1059 (size=12629) 2024-11-13T18:36:28,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523047992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523047995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523048009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523048011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523048116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523048117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523048134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523048134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523048184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,194 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 430b0529634d320a32ba155ab3165681 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:36:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:36:28,283 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/0e5eca012bd0473e97a7bc56e3f4950f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0e5eca012bd0473e97a7bc56e3f4950f 2024-11-13T18:36:28,299 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/B of 430b0529634d320a32ba155ab3165681 into 0e5eca012bd0473e97a7bc56e3f4950f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:28,300 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:28,300 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/B, priority=13, startTime=1731522987731; duration=0sec 2024-11-13T18:36:28,300 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:28,300 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:28,354 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/ebf47cd69a324b28ac0c3342f5b3ebea 2024-11-13T18:36:28,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523048333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523048334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523048354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523048353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/b9e4bc7d526a4388a533bab1e686d21e is 50, key is test_row_0/C:col10/1731522986709/Put/seqid=0 2024-11-13T18:36:28,418 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/581735a1f1f54f96aef018c8f3173860 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/581735a1f1f54f96aef018c8f3173860 2024-11-13T18:36:28,433 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/C of 430b0529634d320a32ba155ab3165681 into 581735a1f1f54f96aef018c8f3173860(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:28,433 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:28,433 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/C, priority=13, startTime=1731522987733; duration=0sec 2024-11-13T18:36:28,434 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:28,434 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:28,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741884_1060 (size=12151) 2024-11-13T18:36:28,447 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/b9e4bc7d526a4388a533bab1e686d21e 2024-11-13T18:36:28,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/baa707b2ae854dfa89574d5f1bd7104f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/baa707b2ae854dfa89574d5f1bd7104f 2024-11-13T18:36:28,465 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/baa707b2ae854dfa89574d5f1bd7104f, entries=150, sequenceid=221, filesize=11.9 K 2024-11-13T18:36:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/ebf47cd69a324b28ac0c3342f5b3ebea as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ebf47cd69a324b28ac0c3342f5b3ebea 2024-11-13T18:36:28,476 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ebf47cd69a324b28ac0c3342f5b3ebea, entries=150, sequenceid=221, filesize=11.9 K 2024-11-13T18:36:28,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/b9e4bc7d526a4388a533bab1e686d21e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b9e4bc7d526a4388a533bab1e686d21e 2024-11-13T18:36:28,496 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b9e4bc7d526a4388a533bab1e686d21e, entries=150, sequenceid=221, filesize=11.9 K 2024-11-13T18:36:28,501 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 430b0529634d320a32ba155ab3165681 in 758ms, sequenceid=221, compaction requested=false 2024-11-13T18:36:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T18:36:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T18:36:28,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T18:36:28,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3780 sec 2024-11-13T18:36:28,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.3900 sec 2024-11-13T18:36:28,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:28,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-13T18:36:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:28,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523048685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523048697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/3daef9fb487b4580aa58480e86ccefed is 50, key is test_row_0/A:col10/1731522987987/Put/seqid=0 2024-11-13T18:36:28,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523048707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523048707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741885_1061 (size=12151) 2024-11-13T18:36:28,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523048824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523048833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:28,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523048842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523049041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523049054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523049056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:29,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/3daef9fb487b4580aa58480e86ccefed 2024-11-13T18:36:29,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523049213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:29,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:29,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/c8f4a223dc2d4e5986020ee4426f9b12 is 50, key is test_row_0/B:col10/1731522987987/Put/seqid=0 2024-11-13T18:36:29,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523049366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:29,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523049366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523049375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:29,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:29,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741886_1062 (size=12151) 2024-11-13T18:36:29,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/c8f4a223dc2d4e5986020ee4426f9b12 2024-11-13T18:36:29,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/7ba1f94fac7c45aebea91a3d6237e804 is 50, key is test_row_0/C:col10/1731522987987/Put/seqid=0 2024-11-13T18:36:29,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741887_1063 (size=12151) 2024-11-13T18:36:29,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/7ba1f94fac7c45aebea91a3d6237e804 2024-11-13T18:36:29,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/3daef9fb487b4580aa58480e86ccefed as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/3daef9fb487b4580aa58480e86ccefed 2024-11-13T18:36:29,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/3daef9fb487b4580aa58480e86ccefed, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T18:36:29,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/c8f4a223dc2d4e5986020ee4426f9b12 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/c8f4a223dc2d4e5986020ee4426f9b12 2024-11-13T18:36:29,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/c8f4a223dc2d4e5986020ee4426f9b12, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T18:36:29,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/7ba1f94fac7c45aebea91a3d6237e804 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7ba1f94fac7c45aebea91a3d6237e804 2024-11-13T18:36:29,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7ba1f94fac7c45aebea91a3d6237e804, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T18:36:29,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 430b0529634d320a32ba155ab3165681 in 975ms, sequenceid=254, compaction requested=true 2024-11-13T18:36:29,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:29,649 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:29,658 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:29,658 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/A is initiating minor compaction (all files) 2024-11-13T18:36:29,659 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/A in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:29,659 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b54efc860ab944cbb9d4e9b41ec1d13f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/baa707b2ae854dfa89574d5f1bd7104f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/3daef9fb487b4580aa58480e86ccefed] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.1 K 2024-11-13T18:36:29,659 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b54efc860ab944cbb9d4e9b41ec1d13f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731522986665 2024-11-13T18:36:29,660 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa707b2ae854dfa89574d5f1bd7104f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731522986696 2024-11-13T18:36:29,660 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3daef9fb487b4580aa58480e86ccefed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731522987961 2024-11-13T18:36:29,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:29,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:29,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:29,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:29,669 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:29,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:29,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:29,671 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:29,671 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/B is initiating minor compaction (all files) 2024-11-13T18:36:29,671 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/B in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:29,671 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0e5eca012bd0473e97a7bc56e3f4950f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ebf47cd69a324b28ac0c3342f5b3ebea, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/c8f4a223dc2d4e5986020ee4426f9b12] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.1 K 2024-11-13T18:36:29,672 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5eca012bd0473e97a7bc56e3f4950f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731522986665 2024-11-13T18:36:29,672 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ebf47cd69a324b28ac0c3342f5b3ebea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731522986696 2024-11-13T18:36:29,673 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f4a223dc2d4e5986020ee4426f9b12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731522987961 2024-11-13T18:36:29,680 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:29,680 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/7b86893f084049fa9d2f55179886f0f6 is 50, key is test_row_0/A:col10/1731522987987/Put/seqid=0 2024-11-13T18:36:29,691 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:29,692 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/e528f169f9354e58b94a27d9c8f6326e is 50, key is test_row_0/B:col10/1731522987987/Put/seqid=0 2024-11-13T18:36:29,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741889_1065 (size=12731) 2024-11-13T18:36:29,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741888_1064 (size=12731) 2024-11-13T18:36:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:30,036 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:30,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T18:36:30,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/9ef3e2ae5e90429f81c5812e05dfb781 is 50, key is test_row_0/A:col10/1731522990004/Put/seqid=0 2024-11-13T18:36:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741890_1066 (size=12301) 2024-11-13T18:36:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523050261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T18:36:30,294 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:30,295 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T18:36:30,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523050264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,301 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:36:30,304 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:30,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523050264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523050286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523050301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523050407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523050408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:36:30,459 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/e528f169f9354e58b94a27d9c8f6326e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/e528f169f9354e58b94a27d9c8f6326e 2024-11-13T18:36:30,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523050425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523050426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,468 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/7b86893f084049fa9d2f55179886f0f6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7b86893f084049fa9d2f55179886f0f6 2024-11-13T18:36:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523050425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,482 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/A of 430b0529634d320a32ba155ab3165681 into 7b86893f084049fa9d2f55179886f0f6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:30,482 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:30,482 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/A, priority=13, startTime=1731522989648; duration=0sec 2024-11-13T18:36:30,482 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:30,482 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:30,484 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:30,485 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:30,486 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/C is initiating minor compaction (all files) 2024-11-13T18:36:30,486 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/C in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:30,486 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/581735a1f1f54f96aef018c8f3173860, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b9e4bc7d526a4388a533bab1e686d21e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7ba1f94fac7c45aebea91a3d6237e804] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.1 K 2024-11-13T18:36:30,486 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 581735a1f1f54f96aef018c8f3173860, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731522986665 2024-11-13T18:36:30,487 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e4bc7d526a4388a533bab1e686d21e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731522986696 2024-11-13T18:36:30,487 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/B of 430b0529634d320a32ba155ab3165681 into e528f169f9354e58b94a27d9c8f6326e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:30,488 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:30,488 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ba1f94fac7c45aebea91a3d6237e804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731522987961 2024-11-13T18:36:30,488 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/B, priority=13, startTime=1731522989669; duration=0sec 2024-11-13T18:36:30,488 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:30,488 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:30,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:36:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:30,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,534 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:30,535 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/12eca725732f45ad9677be402e5cdd07 is 50, key is test_row_0/C:col10/1731522987987/Put/seqid=0 2024-11-13T18:36:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741891_1067 (size=12731) 2024-11-13T18:36:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:36:30,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/9ef3e2ae5e90429f81c5812e05dfb781 2024-11-13T18:36:30,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523050630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,651 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/12eca725732f45ad9677be402e5cdd07 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/12eca725732f45ad9677be402e5cdd07 2024-11-13T18:36:30,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523050632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:36:30,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:30,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:30,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:30,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,704 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/C of 430b0529634d320a32ba155ab3165681 into 12eca725732f45ad9677be402e5cdd07(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:30,704 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:30,704 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/C, priority=13, startTime=1731522989669; duration=0sec 2024-11-13T18:36:30,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/5e901652649543cabdd7f5e6027b7119 is 50, key is test_row_0/B:col10/1731522990004/Put/seqid=0 2024-11-13T18:36:30,709 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:30,709 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:30,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523050678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523050678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523050679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:30,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741892_1068 (size=12301) 2024-11-13T18:36:30,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/5e901652649543cabdd7f5e6027b7119 2024-11-13T18:36:30,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/6767e74fd1e64cc1aeb9686532e40d87 is 50, key is test_row_0/C:col10/1731522990004/Put/seqid=0 2024-11-13T18:36:30,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:36:30,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:30,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:30,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:30,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:30,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741893_1069 (size=12301) 2024-11-13T18:36:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:36:30,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/6767e74fd1e64cc1aeb9686532e40d87 2024-11-13T18:36:30,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/9ef3e2ae5e90429f81c5812e05dfb781 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9ef3e2ae5e90429f81c5812e05dfb781 2024-11-13T18:36:30,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523050955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523050974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9ef3e2ae5e90429f81c5812e05dfb781, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T18:36:31,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/5e901652649543cabdd7f5e6027b7119 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/5e901652649543cabdd7f5e6027b7119 2024-11-13T18:36:31,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/5e901652649543cabdd7f5e6027b7119, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T18:36:31,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/6767e74fd1e64cc1aeb9686532e40d87 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/6767e74fd1e64cc1aeb9686532e40d87 2024-11-13T18:36:31,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523051046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523051047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/6767e74fd1e64cc1aeb9686532e40d87, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T18:36:31,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 430b0529634d320a32ba155ab3165681 in 1026ms, sequenceid=267, compaction requested=false 2024-11-13T18:36:31,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:31,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:31,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T18:36:31,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:36:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/573193edee444a9dab03927013039040 is 50, key is test_row_0/A:col10/1731522990264/Put/seqid=0 2024-11-13T18:36:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741894_1070 (size=14741) 2024-11-13T18:36:31,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/573193edee444a9dab03927013039040 2024-11-13T18:36:31,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/58d5c07a104440fbadd5fe385a7f9301 is 50, key is test_row_0/B:col10/1731522990264/Put/seqid=0 2024-11-13T18:36:31,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:36:31,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:31,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741895_1071 (size=12301) 2024-11-13T18:36:31,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/58d5c07a104440fbadd5fe385a7f9301 2024-11-13T18:36:31,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/fcfb10abfc424e33a00f263daa3fb88a is 50, key is test_row_0/C:col10/1731522990264/Put/seqid=0 2024-11-13T18:36:31,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:36:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:31,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:31,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523051443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741896_1072 (size=12301) 2024-11-13T18:36:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:36:31,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/fcfb10abfc424e33a00f263daa3fb88a 2024-11-13T18:36:31,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/573193edee444a9dab03927013039040 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/573193edee444a9dab03927013039040 2024-11-13T18:36:31,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/573193edee444a9dab03927013039040, entries=200, sequenceid=294, filesize=14.4 K 2024-11-13T18:36:31,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/58d5c07a104440fbadd5fe385a7f9301 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/58d5c07a104440fbadd5fe385a7f9301 2024-11-13T18:36:31,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/58d5c07a104440fbadd5fe385a7f9301, entries=150, sequenceid=294, filesize=12.0 K 2024-11-13T18:36:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/fcfb10abfc424e33a00f263daa3fb88a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/fcfb10abfc424e33a00f263daa3fb88a 2024-11-13T18:36:31,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/fcfb10abfc424e33a00f263daa3fb88a, entries=150, sequenceid=294, filesize=12.0 K 2024-11-13T18:36:31,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 430b0529634d320a32ba155ab3165681 in 457ms, sequenceid=294, compaction requested=true 2024-11-13T18:36:31,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:31,522 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:31,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:31,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:31,523 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:31,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:31,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:31,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:31,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:31,524 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:31,524 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/A is initiating minor compaction (all files) 2024-11-13T18:36:31,524 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/A in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,524 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7b86893f084049fa9d2f55179886f0f6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9ef3e2ae5e90429f81c5812e05dfb781, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/573193edee444a9dab03927013039040] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=38.8 K 2024-11-13T18:36:31,525 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b86893f084049fa9d2f55179886f0f6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731522987961 2024-11-13T18:36:31,525 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:31,525 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/B is initiating minor compaction (all files) 2024-11-13T18:36:31,525 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/B in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,525 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/e528f169f9354e58b94a27d9c8f6326e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/5e901652649543cabdd7f5e6027b7119, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/58d5c07a104440fbadd5fe385a7f9301] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.5 K 2024-11-13T18:36:31,526 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef3e2ae5e90429f81c5812e05dfb781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731522989924 2024-11-13T18:36:31,526 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e528f169f9354e58b94a27d9c8f6326e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731522987961 2024-11-13T18:36:31,526 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 573193edee444a9dab03927013039040, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731522990252 2024-11-13T18:36:31,526 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e901652649543cabdd7f5e6027b7119, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731522989924 2024-11-13T18:36:31,527 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 58d5c07a104440fbadd5fe385a7f9301, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731522990264 2024-11-13T18:36:31,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T18:36:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:31,570 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:36:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:31,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:31,709 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:31,710 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/1970887a03744dae9337cf1eb0ae2465 is 50, key is test_row_0/A:col10/1731522990264/Put/seqid=0 2024-11-13T18:36:31,723 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:31,724 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/37aa650f17f34a259b34624b4709fb74 is 50, key is test_row_0/B:col10/1731522990264/Put/seqid=0 2024-11-13T18:36:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/a6b890b8c1a2408e80eedea62a227174 is 50, key is test_row_0/A:col10/1731522991389/Put/seqid=0 2024-11-13T18:36:31,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523051835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523051837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523051842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523051852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523051862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523051994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:31,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523051994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523051995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523051995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523051995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741897_1073 (size=12983) 2024-11-13T18:36:32,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741899_1075 (size=12301) 2024-11-13T18:36:32,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741898_1074 (size=12983) 2024-11-13T18:36:32,051 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/a6b890b8c1a2408e80eedea62a227174 2024-11-13T18:36:32,068 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/37aa650f17f34a259b34624b4709fb74 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/37aa650f17f34a259b34624b4709fb74 2024-11-13T18:36:32,075 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/1970887a03744dae9337cf1eb0ae2465 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/1970887a03744dae9337cf1eb0ae2465 2024-11-13T18:36:32,110 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/A of 430b0529634d320a32ba155ab3165681 into 1970887a03744dae9337cf1eb0ae2465(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:32,110 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/B of 430b0529634d320a32ba155ab3165681 into 37aa650f17f34a259b34624b4709fb74(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:32,110 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:32,110 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:32,110 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/A, priority=13, startTime=1731522991522; duration=0sec 2024-11-13T18:36:32,110 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/B, priority=13, startTime=1731522991523; duration=0sec 2024-11-13T18:36:32,110 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:32,110 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:32,110 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:32,111 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:32,111 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:32,112 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:32,112 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/C is initiating minor compaction (all files) 2024-11-13T18:36:32,112 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/C in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:32,112 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/12eca725732f45ad9677be402e5cdd07, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/6767e74fd1e64cc1aeb9686532e40d87, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/fcfb10abfc424e33a00f263daa3fb88a] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.5 K 2024-11-13T18:36:32,113 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12eca725732f45ad9677be402e5cdd07, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731522987961 2024-11-13T18:36:32,113 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6767e74fd1e64cc1aeb9686532e40d87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731522989924 2024-11-13T18:36:32,114 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcfb10abfc424e33a00f263daa3fb88a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731522990264 2024-11-13T18:36:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/94849c30102544099c364402845a83f0 is 50, key is test_row_0/B:col10/1731522991389/Put/seqid=0 2024-11-13T18:36:32,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523052204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523052213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523052214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523052214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523052214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,227 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:32,228 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/7b0fe3e846884d76979fc00aeb60f7b8 is 50, key is test_row_0/C:col10/1731522990264/Put/seqid=0 2024-11-13T18:36:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:36:32,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523052525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523052534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523052534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523052534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523052534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:32,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741900_1076 (size=12301) 2024-11-13T18:36:32,585 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/94849c30102544099c364402845a83f0 2024-11-13T18:36:32,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741901_1077 (size=12983) 2024-11-13T18:36:32,693 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/7b0fe3e846884d76979fc00aeb60f7b8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7b0fe3e846884d76979fc00aeb60f7b8 2024-11-13T18:36:32,701 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/C of 430b0529634d320a32ba155ab3165681 into 7b0fe3e846884d76979fc00aeb60f7b8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:32,701 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:32,701 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/C, priority=13, startTime=1731522991523; duration=0sec 2024-11-13T18:36:32,701 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:32,701 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:32,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ad59f7798e8a48e2b277191e2e70935c is 50, key is test_row_0/C:col10/1731522991389/Put/seqid=0 2024-11-13T18:36:32,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741902_1078 (size=12301) 2024-11-13T18:36:32,821 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ad59f7798e8a48e2b277191e2e70935c 2024-11-13T18:36:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/a6b890b8c1a2408e80eedea62a227174 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a6b890b8c1a2408e80eedea62a227174 2024-11-13T18:36:32,938 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a6b890b8c1a2408e80eedea62a227174, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T18:36:32,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/94849c30102544099c364402845a83f0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/94849c30102544099c364402845a83f0 2024-11-13T18:36:32,963 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/94849c30102544099c364402845a83f0, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T18:36:32,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ad59f7798e8a48e2b277191e2e70935c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ad59f7798e8a48e2b277191e2e70935c 2024-11-13T18:36:33,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523053043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523053065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523053065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523053065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523053065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:33,125 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ad59f7798e8a48e2b277191e2e70935c, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T18:36:33,126 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 430b0529634d320a32ba155ab3165681 in 1556ms, sequenceid=306, compaction requested=false 2024-11-13T18:36:33,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:33,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T18:36:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T18:36:33,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T18:36:33,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8270 sec 2024-11-13T18:36:33,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.8400 sec 2024-11-13T18:36:34,100 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:34,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:36:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:34,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/232d9175ec1f441ea6cd0669d0b0187a is 50, key is test_row_0/A:col10/1731522994096/Put/seqid=0 2024-11-13T18:36:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523054133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523054142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741903_1079 (size=14741) 2024-11-13T18:36:34,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523054175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523054179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523054209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523054318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523054318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523054324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523054325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523054324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T18:36:34,505 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:34,505 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T18:36:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:36:34,511 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:34,512 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:34,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:34,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/232d9175ec1f441ea6cd0669d0b0187a 2024-11-13T18:36:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:36:34,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:34,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:34,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/0688f052f062499280a2d6cff3bdcc8a is 50, key is test_row_0/B:col10/1731522994096/Put/seqid=0 2024-11-13T18:36:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523054644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523054663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523054664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523054674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523054674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:34,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:34,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741904_1080 (size=12301) 2024-11-13T18:36:34,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:34,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:34,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:36:34,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:34,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523054998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:35,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:36:35,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523055033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:35,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523055049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:35,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523055050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:35,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/0688f052f062499280a2d6cff3bdcc8a 2024-11-13T18:36:35,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523055033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:35,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5 is 50, key is test_row_0/C:col10/1731522994096/Put/seqid=0 2024-11-13T18:36:35,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:35,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741905_1081 (size=12301) 2024-11-13T18:36:35,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5 2024-11-13T18:36:35,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/232d9175ec1f441ea6cd0669d0b0187a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/232d9175ec1f441ea6cd0669d0b0187a 2024-11-13T18:36:35,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/232d9175ec1f441ea6cd0669d0b0187a, entries=200, sequenceid=334, filesize=14.4 K 2024-11-13T18:36:35,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/0688f052f062499280a2d6cff3bdcc8a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0688f052f062499280a2d6cff3bdcc8a 2024-11-13T18:36:35,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0688f052f062499280a2d6cff3bdcc8a, entries=150, sequenceid=334, filesize=12.0 K 2024-11-13T18:36:35,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5 2024-11-13T18:36:35,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5, entries=150, sequenceid=334, filesize=12.0 K 2024-11-13T18:36:35,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 430b0529634d320a32ba155ab3165681 in 1515ms, sequenceid=334, compaction requested=true 2024-11-13T18:36:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:35,616 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:35,617 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:35,618 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/A is initiating minor compaction (all files) 2024-11-13T18:36:35,618 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/A in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,618 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/1970887a03744dae9337cf1eb0ae2465, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a6b890b8c1a2408e80eedea62a227174, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/232d9175ec1f441ea6cd0669d0b0187a] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=39.1 K 2024-11-13T18:36:35,618 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1970887a03744dae9337cf1eb0ae2465, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731522990264 2024-11-13T18:36:35,619 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6b890b8c1a2408e80eedea62a227174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731522991171 2024-11-13T18:36:35,619 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 232d9175ec1f441ea6cd0669d0b0187a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731522991833 2024-11-13T18:36:35,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:35,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:35,632 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:35,634 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:35,634 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/B is initiating minor compaction (all files) 2024-11-13T18:36:35,634 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/B in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,634 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/37aa650f17f34a259b34624b4709fb74, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/94849c30102544099c364402845a83f0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0688f052f062499280a2d6cff3bdcc8a] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.7 K 2024-11-13T18:36:35,635 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 37aa650f17f34a259b34624b4709fb74, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731522990264 2024-11-13T18:36:35,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:35,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:35,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:35,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:35,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:35,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:36:35,636 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 94849c30102544099c364402845a83f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731522991171 2024-11-13T18:36:35,637 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0688f052f062499280a2d6cff3bdcc8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731522991833 2024-11-13T18:36:35,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:36:35,689 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:35,690 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/d567ea0544ea488ba912a51f076ba81b is 50, key is test_row_0/A:col10/1731522994096/Put/seqid=0 2024-11-13T18:36:35,719 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:35,720 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/b8d82c67d9284869bda12076ae4ca862 is 50, key is test_row_0/B:col10/1731522994096/Put/seqid=0 2024-11-13T18:36:35,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:35,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:35,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/8d78efa4809b48a483fb0418f443dd99 is 50, key is test_row_0/A:col10/1731522994176/Put/seqid=0 2024-11-13T18:36:36,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741907_1083 (size=13085) 2024-11-13T18:36:36,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741908_1084 (size=14741) 2024-11-13T18:36:36,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/8d78efa4809b48a483fb0418f443dd99 2024-11-13T18:36:36,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741906_1082 (size=13085) 2024-11-13T18:36:36,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:36,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523056256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/9c619e0b56cf4bd08d72e634453e4ae7 is 50, key is test_row_0/B:col10/1731522994176/Put/seqid=0 2024-11-13T18:36:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523056261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523056310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523056313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523056297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:36,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:36,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,546 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/b8d82c67d9284869bda12076ae4ca862 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b8d82c67d9284869bda12076ae4ca862 2024-11-13T18:36:36,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523056474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523056474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523056513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523056514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523056524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,568 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/d567ea0544ea488ba912a51f076ba81b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/d567ea0544ea488ba912a51f076ba81b 2024-11-13T18:36:36,607 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/A of 430b0529634d320a32ba155ab3165681 into d567ea0544ea488ba912a51f076ba81b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:36,608 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:36,608 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/A, priority=13, startTime=1731522995616; duration=0sec 2024-11-13T18:36:36,608 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:36,608 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:36,608 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:36,609 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/B of 430b0529634d320a32ba155ab3165681 into b8d82c67d9284869bda12076ae4ca862(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:36,609 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:36,609 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/B, priority=13, startTime=1731522995632; duration=0sec 2024-11-13T18:36:36,610 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:36,610 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:36,610 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:36,610 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/C is initiating minor compaction (all files) 2024-11-13T18:36:36,610 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/C in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,610 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7b0fe3e846884d76979fc00aeb60f7b8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ad59f7798e8a48e2b277191e2e70935c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=36.7 K 2024-11-13T18:36:36,611 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0fe3e846884d76979fc00aeb60f7b8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731522990264 2024-11-13T18:36:36,611 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad59f7798e8a48e2b277191e2e70935c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731522991171 2024-11-13T18:36:36,612 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee4d7c78ea5d4ac0b006a79a8b08f7f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731522991833 2024-11-13T18:36:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:36:36,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741909_1085 (size=12301) 2024-11-13T18:36:36,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:36,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/9c619e0b56cf4bd08d72e634453e4ae7 2024-11-13T18:36:36,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:36,726 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:36,727 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/32fbe7c9aefa46f8aad758ba92a6bdc5 is 50, key is test_row_0/C:col10/1731522994096/Put/seqid=0 2024-11-13T18:36:36,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523056764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/a8fa39bd0dcf49458d67ddbe2898cdc2 is 50, key is test_row_0/C:col10/1731522994176/Put/seqid=0 2024-11-13T18:36:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523056764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523056765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523056765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:36,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523056773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:37,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:37,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741911_1087 (size=12301) 2024-11-13T18:36:37,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523057073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/a8fa39bd0dcf49458d67ddbe2898cdc2 2024-11-13T18:36:37,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/8d78efa4809b48a483fb0418f443dd99 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/8d78efa4809b48a483fb0418f443dd99 2024-11-13T18:36:37,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/8d78efa4809b48a483fb0418f443dd99, entries=200, sequenceid=346, filesize=14.4 K 2024-11-13T18:36:37,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/9c619e0b56cf4bd08d72e634453e4ae7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/9c619e0b56cf4bd08d72e634453e4ae7 2024-11-13T18:36:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523057084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523057086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523057086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/9c619e0b56cf4bd08d72e634453e4ae7, entries=150, sequenceid=346, filesize=12.0 K 2024-11-13T18:36:37,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/a8fa39bd0dcf49458d67ddbe2898cdc2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a8fa39bd0dcf49458d67ddbe2898cdc2 2024-11-13T18:36:37,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a8fa39bd0dcf49458d67ddbe2898cdc2, entries=150, sequenceid=346, filesize=12.0 K 2024-11-13T18:36:37,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 430b0529634d320a32ba155ab3165681 in 1540ms, sequenceid=346, compaction requested=false 2024-11-13T18:36:37,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T18:36:37,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:37,292 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T18:36:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:37,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:37,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741910_1086 (size=13085) 2024-11-13T18:36:37,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/b0684162366248ce8d46f5811a156652 is 50, key is test_row_0/A:col10/1731522996132/Put/seqid=0 2024-11-13T18:36:37,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741912_1088 (size=12301) 2024-11-13T18:36:37,327 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/b0684162366248ce8d46f5811a156652 2024-11-13T18:36:37,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/2fac7b2a623c42eea2987d70b2ff7dc6 is 50, key is test_row_0/B:col10/1731522996132/Put/seqid=0 2024-11-13T18:36:37,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741913_1089 (size=12301) 2024-11-13T18:36:37,450 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/2fac7b2a623c42eea2987d70b2ff7dc6 2024-11-13T18:36:37,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/25e9b8ad703844ad863621af24bac97b is 50, key is test_row_0/C:col10/1731522996132/Put/seqid=0 2024-11-13T18:36:37,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523057530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741914_1090 (size=12301) 2024-11-13T18:36:37,608 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/25e9b8ad703844ad863621af24bac97b 2024-11-13T18:36:37,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523057664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/b0684162366248ce8d46f5811a156652 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b0684162366248ce8d46f5811a156652 2024-11-13T18:36:37,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523057664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523057684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,744 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b0684162366248ce8d46f5811a156652, entries=150, sequenceid=373, filesize=12.0 K 2024-11-13T18:36:37,748 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/32fbe7c9aefa46f8aad758ba92a6bdc5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/32fbe7c9aefa46f8aad758ba92a6bdc5 2024-11-13T18:36:37,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/2fac7b2a623c42eea2987d70b2ff7dc6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/2fac7b2a623c42eea2987d70b2ff7dc6 2024-11-13T18:36:37,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523057664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523057728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:37,766 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 430b0529634d320a32ba155ab3165681/C of 430b0529634d320a32ba155ab3165681 into 32fbe7c9aefa46f8aad758ba92a6bdc5(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-13T18:36:37,766 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:37,766 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/C, priority=13, startTime=1731522995636; duration=1sec 2024-11-13T18:36:37,766 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:37,766 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:37,810 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/2fac7b2a623c42eea2987d70b2ff7dc6, entries=150, sequenceid=373, filesize=12.0 K 2024-11-13T18:36:37,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/25e9b8ad703844ad863621af24bac97b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/25e9b8ad703844ad863621af24bac97b 2024-11-13T18:36:37,840 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/25e9b8ad703844ad863621af24bac97b, entries=150, sequenceid=373, filesize=12.0 K 2024-11-13T18:36:37,890 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 430b0529634d320a32ba155ab3165681 in 599ms, sequenceid=373, compaction requested=true 2024-11-13T18:36:37,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:37,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:37,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T18:36:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T18:36:37,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T18:36:37,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 3.3910 sec 2024-11-13T18:36:37,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3810 sec 2024-11-13T18:36:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:37,986 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:37,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:36:38,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd is 50, key is test_row_0/A:col10/1731522997517/Put/seqid=0 2024-11-13T18:36:38,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741915_1091 (size=14741) 2024-11-13T18:36:38,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd 2024-11-13T18:36:38,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/99b62ccabcbe49a8ba9742345772dd9a is 50, key is test_row_0/B:col10/1731522997517/Put/seqid=0 2024-11-13T18:36:38,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741916_1092 (size=12301) 2024-11-13T18:36:38,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/99b62ccabcbe49a8ba9742345772dd9a 2024-11-13T18:36:38,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/1c137a0c270b4081902d8c17b07ddc58 is 50, key is test_row_0/C:col10/1731522997517/Put/seqid=0 2024-11-13T18:36:38,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741917_1093 (size=12301) 2024-11-13T18:36:38,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/1c137a0c270b4081902d8c17b07ddc58 2024-11-13T18:36:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T18:36:38,698 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:38,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T18:36:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:36:38,703 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:38,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd 2024-11-13T18:36:38,704 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:38,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:38,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd, entries=200, sequenceid=386, filesize=14.4 K 2024-11-13T18:36:38,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/99b62ccabcbe49a8ba9742345772dd9a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/99b62ccabcbe49a8ba9742345772dd9a 2024-11-13T18:36:38,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/99b62ccabcbe49a8ba9742345772dd9a, entries=150, sequenceid=386, filesize=12.0 K 2024-11-13T18:36:38,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/1c137a0c270b4081902d8c17b07ddc58 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1c137a0c270b4081902d8c17b07ddc58 2024-11-13T18:36:38,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1c137a0c270b4081902d8c17b07ddc58, entries=150, sequenceid=386, filesize=12.0 K 2024-11-13T18:36:38,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=114.05 KB/116790 for 430b0529634d320a32ba155ab3165681 in 772ms, sequenceid=386, compaction requested=true 2024-11-13T18:36:38,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:38,759 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:38,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:38,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:38,760 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:38,761 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:38,761 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/A is initiating minor compaction (all files) 2024-11-13T18:36:38,761 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/A in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:38,761 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/d567ea0544ea488ba912a51f076ba81b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/8d78efa4809b48a483fb0418f443dd99, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b0684162366248ce8d46f5811a156652, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=53.6 K 2024-11-13T18:36:38,762 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d567ea0544ea488ba912a51f076ba81b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731522991833 2024-11-13T18:36:38,763 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d78efa4809b48a483fb0418f443dd99, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522994123 2024-11-13T18:36:38,763 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:38,763 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/B is initiating minor compaction (all files) 2024-11-13T18:36:38,763 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/B in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:38,764 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b8d82c67d9284869bda12076ae4ca862, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/9c619e0b56cf4bd08d72e634453e4ae7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/2fac7b2a623c42eea2987d70b2ff7dc6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/99b62ccabcbe49a8ba9742345772dd9a] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=48.8 K 2024-11-13T18:36:38,764 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0684162366248ce8d46f5811a156652, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731522996132 2024-11-13T18:36:38,764 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d82c67d9284869bda12076ae4ca862, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731522991833 2024-11-13T18:36:38,764 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af28e46a9ef4bdb8ce8c2ba7e0d09cd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731522997437 2024-11-13T18:36:38,765 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c619e0b56cf4bd08d72e634453e4ae7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522994176 2024-11-13T18:36:38,770 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fac7b2a623c42eea2987d70b2ff7dc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731522996132 2024-11-13T18:36:38,774 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b62ccabcbe49a8ba9742345772dd9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731522997500 2024-11-13T18:36:38,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:38,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:38,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:38,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:36:38,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T18:36:38,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:38,867 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T18:36:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. as already flushing 2024-11-13T18:36:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:38,949 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:38,950 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/e0bd5adfe4ec454ca178a314c3fe83e7 is 50, key is test_row_0/B:col10/1731522997517/Put/seqid=0 2024-11-13T18:36:38,965 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:38,966 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/c27b7cfed1d949e7aacff58becac9891 is 50, key is test_row_0/A:col10/1731522997517/Put/seqid=0 2024-11-13T18:36:39,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/949a680cfb134bf4966825aa6e84794a is 50, key is test_row_0/A:col10/1731522998709/Put/seqid=0 2024-11-13T18:36:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:36:39,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523059093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523059099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523059138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523059141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523059160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523059274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523059274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741918_1094 (size=13221) 2024-11-13T18:36:39,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523059283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523059283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523059295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:36:39,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741920_1096 (size=14741) 2024-11-13T18:36:39,360 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/949a680cfb134bf4966825aa6e84794a 2024-11-13T18:36:39,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741919_1095 (size=13221) 2024-11-13T18:36:39,469 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/c27b7cfed1d949e7aacff58becac9891 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/c27b7cfed1d949e7aacff58becac9891 2024-11-13T18:36:39,479 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/A of 430b0529634d320a32ba155ab3165681 into c27b7cfed1d949e7aacff58becac9891(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:39,479 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:39,479 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/A, priority=12, startTime=1731522998759; duration=0sec 2024-11-13T18:36:39,482 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:39,482 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:39,482 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:36:39,484 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:36:39,484 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 430b0529634d320a32ba155ab3165681/C is initiating minor compaction (all files) 2024-11-13T18:36:39,484 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 430b0529634d320a32ba155ab3165681/C in TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:39,484 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/32fbe7c9aefa46f8aad758ba92a6bdc5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a8fa39bd0dcf49458d67ddbe2898cdc2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/25e9b8ad703844ad863621af24bac97b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1c137a0c270b4081902d8c17b07ddc58] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp, totalSize=48.8 K 2024-11-13T18:36:39,485 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32fbe7c9aefa46f8aad758ba92a6bdc5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731522991833 2024-11-13T18:36:39,501 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8fa39bd0dcf49458d67ddbe2898cdc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731522994176 2024-11-13T18:36:39,533 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25e9b8ad703844ad863621af24bac97b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731522996132 2024-11-13T18:36:39,534 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c137a0c270b4081902d8c17b07ddc58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731522997500 2024-11-13T18:36:39,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/b07b52905df8410daebd3e2f79e051f5 is 50, key is test_row_0/B:col10/1731522998709/Put/seqid=0 2024-11-13T18:36:39,695 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 430b0529634d320a32ba155ab3165681#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:39,696 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/80c00858d793456ea8a8520c68ca2210 is 50, key is test_row_0/C:col10/1731522997517/Put/seqid=0 2024-11-13T18:36:39,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523059510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523059528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523059528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523059529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523059528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:39,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741921_1097 (size=12301) 2024-11-13T18:36:39,705 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/b07b52905df8410daebd3e2f79e051f5 2024-11-13T18:36:39,726 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/e0bd5adfe4ec454ca178a314c3fe83e7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/e0bd5adfe4ec454ca178a314c3fe83e7 2024-11-13T18:36:39,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/d11f197e23524d3a9acd12bcd730b284 is 50, key is test_row_0/C:col10/1731522998709/Put/seqid=0 2024-11-13T18:36:39,736 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/B of 430b0529634d320a32ba155ab3165681 into e0bd5adfe4ec454ca178a314c3fe83e7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:39,736 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:39,736 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/B, priority=12, startTime=1731522998760; duration=0sec 2024-11-13T18:36:39,736 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:39,736 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:39,773 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T18:36:39,773 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:39,780 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:39,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:39,782 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:39,805 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T18:36:39,806 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:39,806 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:39,806 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:39,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:36:39,935 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T18:36:39,935 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:39,935 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:39,935 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:39,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:39,954 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T18:36:39,954 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:39,954 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:39,954 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:39,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:40,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741922_1098 (size=13221) 2024-11-13T18:36:40,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:40,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58536 deadline: 1731523060004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58470 deadline: 1731523060005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:40,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58480 deadline: 1731523060006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:40,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58496 deadline: 1731523060014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:40,022 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/80c00858d793456ea8a8520c68ca2210 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/80c00858d793456ea8a8520c68ca2210 2024-11-13T18:36:40,031 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 430b0529634d320a32ba155ab3165681/C of 430b0529634d320a32ba155ab3165681 into 80c00858d793456ea8a8520c68ca2210(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:40,031 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:40,031 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., storeName=430b0529634d320a32ba155ab3165681/C, priority=12, startTime=1731522998777; duration=0sec 2024-11-13T18:36:40,032 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:40,032 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:40,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58510 deadline: 1731523060033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:36:40,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741923_1099 (size=12301) 2024-11-13T18:36:40,246 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/d11f197e23524d3a9acd12bcd730b284 2024-11-13T18:36:40,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/949a680cfb134bf4966825aa6e84794a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/949a680cfb134bf4966825aa6e84794a 2024-11-13T18:36:40,272 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/949a680cfb134bf4966825aa6e84794a, entries=200, sequenceid=406, filesize=14.4 K 2024-11-13T18:36:40,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/b07b52905df8410daebd3e2f79e051f5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b07b52905df8410daebd3e2f79e051f5 2024-11-13T18:36:40,285 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b07b52905df8410daebd3e2f79e051f5, entries=150, sequenceid=406, filesize=12.0 K 2024-11-13T18:36:40,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/d11f197e23524d3a9acd12bcd730b284 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/d11f197e23524d3a9acd12bcd730b284 2024-11-13T18:36:40,292 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/d11f197e23524d3a9acd12bcd730b284, entries=150, sequenceid=406, filesize=12.0 K 2024-11-13T18:36:40,393 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 430b0529634d320a32ba155ab3165681 in 1518ms, sequenceid=406, compaction requested=false 2024-11-13T18:36:40,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:40,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:40,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T18:36:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T18:36:40,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T18:36:40,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6920 sec 2024-11-13T18:36:40,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7010 sec 2024-11-13T18:36:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:40,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T18:36:40,519 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T18:36:40,519 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:40,519 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,519 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T18:36:40,519 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,519 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:40,519 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,519 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:40,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:40,522 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T18:36:40,522 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:40,522 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,522 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:40,527 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T18:36:40,527 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:40,527 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,527 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:40,546 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T18:36:40,546 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:36:40,546 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,546 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:40,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/ae7d26c20eb64b1baf2e279eab0c1642 is 50, key is test_row_0/A:col10/1731523000515/Put/seqid=0 2024-11-13T18:36:40,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741924_1100 (size=12301) 2024-11-13T18:36:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T18:36:40,854 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:40,854 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1067 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1102 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 537 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1607 rows 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 535 2024-11-13T18:36:40,855 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1604 rows 2024-11-13T18:36:40,856 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:36:40,856 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:36:40,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:40,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:36:40,861 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:36:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:40,880 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523000879"}]},"ts":"1731523000879"} 2024-11-13T18:36:40,883 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:36:40,883 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:36:40,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:36:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:36:40,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, UNASSIGN}] 2024-11-13T18:36:40,892 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, UNASSIGN 2024-11-13T18:36:40,894 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=430b0529634d320a32ba155ab3165681, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:40,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, UNASSIGN because future has completed 2024-11-13T18:36:40,922 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:36:40,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:36:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:36:41,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/ae7d26c20eb64b1baf2e279eab0c1642 2024-11-13T18:36:41,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/dee93140d64541af9d8f59b4e8e50fc9 is 50, key is test_row_0/B:col10/1731523000515/Put/seqid=0 2024-11-13T18:36:41,114 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:41,114 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:36:41,115 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing 430b0529634d320a32ba155ab3165681, disabling compactions & flushes 2024-11-13T18:36:41,115 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:36:41,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741925_1101 (size=12301) 2024-11-13T18:36:41,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/dee93140d64541af9d8f59b4e8e50fc9 2024-11-13T18:36:41,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/87c2126c107e41b89b80a0749c46c18d is 50, key is test_row_0/C:col10/1731523000515/Put/seqid=0 2024-11-13T18:36:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:36:41,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741926_1102 (size=12301) 2024-11-13T18:36:41,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/87c2126c107e41b89b80a0749c46c18d 2024-11-13T18:36:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/ae7d26c20eb64b1baf2e279eab0c1642 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ae7d26c20eb64b1baf2e279eab0c1642 2024-11-13T18:36:41,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ae7d26c20eb64b1baf2e279eab0c1642, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T18:36:41,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/dee93140d64541af9d8f59b4e8e50fc9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/dee93140d64541af9d8f59b4e8e50fc9 2024-11-13T18:36:41,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/dee93140d64541af9d8f59b4e8e50fc9, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T18:36:41,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/87c2126c107e41b89b80a0749c46c18d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/87c2126c107e41b89b80a0749c46c18d 2024-11-13T18:36:41,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/87c2126c107e41b89b80a0749c46c18d, entries=150, sequenceid=428, filesize=12.0 K 2024-11-13T18:36:41,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=13.42 KB/13740 for 430b0529634d320a32ba155ab3165681 in 1190ms, sequenceid=428, compaction requested=true 2024-11-13T18:36:41,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 430b0529634d320a32ba155ab3165681: 2024-11-13T18:36:41,709 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. because compaction request was cancelled 2024-11-13T18:36:41,709 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:A 2024-11-13T18:36:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:A, priority=-2147483648, current under compaction store size is 0 2024-11-13T18:36:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:41,709 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. because compaction request was cancelled 2024-11-13T18:36:41,709 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:B 2024-11-13T18:36:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:B, priority=-2147483648, current under compaction store size is 0 2024-11-13T18:36:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:41,709 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. because compaction request was cancelled 2024-11-13T18:36:41,709 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 430b0529634d320a32ba155ab3165681:C 2024-11-13T18:36:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 430b0529634d320a32ba155ab3165681:C, priority=-2147483648, current under compaction store size is 0 2024-11-13T18:36:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:41,708 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:41,731 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:41,731 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. after waiting 0 ms 2024-11-13T18:36:41,731 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:41,732 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing 430b0529634d320a32ba155ab3165681 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-13T18:36:41,859 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/64ef5ad875574a81b1d23ccd71937c55 is 50, key is test_row_0/A:col10/1731523000544/Put/seqid=0 2024-11-13T18:36:41,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741927_1103 (size=9857) 2024-11-13T18:36:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:36:42,270 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/64ef5ad875574a81b1d23ccd71937c55 2024-11-13T18:36:42,303 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/ef3018f15e4140e8b194ed55102d2196 is 50, key is test_row_0/B:col10/1731523000544/Put/seqid=0 2024-11-13T18:36:42,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741928_1104 (size=9857) 2024-11-13T18:36:42,502 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/ef3018f15e4140e8b194ed55102d2196 2024-11-13T18:36:42,727 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/c48cfd99f9c249fa97c793b2f104ca9b is 50, key is test_row_0/C:col10/1731523000544/Put/seqid=0 2024-11-13T18:36:42,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741929_1105 (size=9857) 2024-11-13T18:36:42,853 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/c48cfd99f9c249fa97c793b2f104ca9b 2024-11-13T18:36:42,875 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/A/64ef5ad875574a81b1d23ccd71937c55 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/64ef5ad875574a81b1d23ccd71937c55 2024-11-13T18:36:42,884 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/64ef5ad875574a81b1d23ccd71937c55, entries=100, sequenceid=433, filesize=9.6 K 2024-11-13T18:36:42,886 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/B/ef3018f15e4140e8b194ed55102d2196 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ef3018f15e4140e8b194ed55102d2196 2024-11-13T18:36:42,901 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ef3018f15e4140e8b194ed55102d2196, entries=100, sequenceid=433, filesize=9.6 K 2024-11-13T18:36:42,902 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/.tmp/C/c48cfd99f9c249fa97c793b2f104ca9b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c48cfd99f9c249fa97c793b2f104ca9b 2024-11-13T18:36:42,910 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c48cfd99f9c249fa97c793b2f104ca9b, entries=100, sequenceid=433, filesize=9.6 K 2024-11-13T18:36:42,981 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 430b0529634d320a32ba155ab3165681 in 1250ms, sequenceid=433, compaction requested=true 2024-11-13T18:36:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:36:43,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9bed7856d84f4edabea8cc7672c63afc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/41d3b1a80d024cada3807840e471a9b2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/835ac9e5b6ff41d99d78e44d75261c2f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95973431cd5645f58317be9448eb53f3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/feb2f77388ca4e19be98b580505bde27, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ddd9e496bc3f4adbacf5bdd96bd68353, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ed4f08e0d5d14022805b303891100bd0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95972265564645eaa5a797aed5eed7f3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/71a1dccc655a4e04b5372256e7cf4d4d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/6d0b0e5e574540e99c9508237d48aaa1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7453ed8f6ad641548eff07f4b0c68916, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b9ef998030224b70bd708f1fc1f683b8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a51859e9997e4edcbab81ea8afd1b0f2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b54efc860ab944cbb9d4e9b41ec1d13f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/f24c2003026543ecb4d00795446aaee5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/baa707b2ae854dfa89574d5f1bd7104f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7b86893f084049fa9d2f55179886f0f6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/3daef9fb487b4580aa58480e86ccefed, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9ef3e2ae5e90429f81c5812e05dfb781, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/573193edee444a9dab03927013039040, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/1970887a03744dae9337cf1eb0ae2465, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a6b890b8c1a2408e80eedea62a227174, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/232d9175ec1f441ea6cd0669d0b0187a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/d567ea0544ea488ba912a51f076ba81b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/8d78efa4809b48a483fb0418f443dd99, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b0684162366248ce8d46f5811a156652, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd] to archive 2024-11-13T18:36:43,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:36:43,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9bed7856d84f4edabea8cc7672c63afc to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9bed7856d84f4edabea8cc7672c63afc 2024-11-13T18:36:43,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/41d3b1a80d024cada3807840e471a9b2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/41d3b1a80d024cada3807840e471a9b2 2024-11-13T18:36:43,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/835ac9e5b6ff41d99d78e44d75261c2f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/835ac9e5b6ff41d99d78e44d75261c2f 2024-11-13T18:36:43,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95973431cd5645f58317be9448eb53f3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95973431cd5645f58317be9448eb53f3 2024-11-13T18:36:43,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/feb2f77388ca4e19be98b580505bde27 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/feb2f77388ca4e19be98b580505bde27 2024-11-13T18:36:43,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ddd9e496bc3f4adbacf5bdd96bd68353 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ddd9e496bc3f4adbacf5bdd96bd68353 2024-11-13T18:36:43,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ed4f08e0d5d14022805b303891100bd0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ed4f08e0d5d14022805b303891100bd0 2024-11-13T18:36:43,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95972265564645eaa5a797aed5eed7f3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/95972265564645eaa5a797aed5eed7f3 2024-11-13T18:36:43,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/71a1dccc655a4e04b5372256e7cf4d4d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/71a1dccc655a4e04b5372256e7cf4d4d 2024-11-13T18:36:43,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/6d0b0e5e574540e99c9508237d48aaa1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/6d0b0e5e574540e99c9508237d48aaa1 2024-11-13T18:36:43,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7453ed8f6ad641548eff07f4b0c68916 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7453ed8f6ad641548eff07f4b0c68916 2024-11-13T18:36:43,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b9ef998030224b70bd708f1fc1f683b8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b9ef998030224b70bd708f1fc1f683b8 2024-11-13T18:36:43,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a51859e9997e4edcbab81ea8afd1b0f2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a51859e9997e4edcbab81ea8afd1b0f2 2024-11-13T18:36:43,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b54efc860ab944cbb9d4e9b41ec1d13f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b54efc860ab944cbb9d4e9b41ec1d13f 2024-11-13T18:36:43,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/f24c2003026543ecb4d00795446aaee5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/f24c2003026543ecb4d00795446aaee5 2024-11-13T18:36:43,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/baa707b2ae854dfa89574d5f1bd7104f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/baa707b2ae854dfa89574d5f1bd7104f 2024-11-13T18:36:43,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7b86893f084049fa9d2f55179886f0f6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/7b86893f084049fa9d2f55179886f0f6 2024-11-13T18:36:43,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/3daef9fb487b4580aa58480e86ccefed to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/3daef9fb487b4580aa58480e86ccefed 2024-11-13T18:36:43,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9ef3e2ae5e90429f81c5812e05dfb781 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/9ef3e2ae5e90429f81c5812e05dfb781 2024-11-13T18:36:43,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/573193edee444a9dab03927013039040 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/573193edee444a9dab03927013039040 2024-11-13T18:36:43,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/1970887a03744dae9337cf1eb0ae2465 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/1970887a03744dae9337cf1eb0ae2465 2024-11-13T18:36:43,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a6b890b8c1a2408e80eedea62a227174 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/a6b890b8c1a2408e80eedea62a227174 2024-11-13T18:36:43,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/232d9175ec1f441ea6cd0669d0b0187a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/232d9175ec1f441ea6cd0669d0b0187a 2024-11-13T18:36:43,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/d567ea0544ea488ba912a51f076ba81b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/d567ea0544ea488ba912a51f076ba81b 2024-11-13T18:36:43,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/8d78efa4809b48a483fb0418f443dd99 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/8d78efa4809b48a483fb0418f443dd99 2024-11-13T18:36:43,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b0684162366248ce8d46f5811a156652 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/b0684162366248ce8d46f5811a156652 2024-11-13T18:36:43,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/0af28e46a9ef4bdb8ce8c2ba7e0d09cd 2024-11-13T18:36:43,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/699ba810500941108d6fc925601c26d0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa9734218de8459f82ada23061493d5f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/463914939a2c436581149a3ef5c00bf1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/71857942fb5f483393f5f36bdd8619e2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa4b00398fc042fc878b63cd949c4ac9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/48e808bf306f498c85e2f204f661fda6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/55535ec929bb43a8819a4fafaaabb258, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/285fa4c23a4d47a59136f6f7974dc55c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/947095edbca0457c9eef01b2e45f6059, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/a2c1462d03a042c781f5d0bfb4270bbe, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/1d08f51bdbe94fdb93f31e3618bbfd4a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0bb983c92fac4742bb4431ce671c60d3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/440650511d5c4d6992940a6bb7e22525, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0e5eca012bd0473e97a7bc56e3f4950f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/debdff204a78404f96d2b158e5217e6b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ebf47cd69a324b28ac0c3342f5b3ebea, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/e528f169f9354e58b94a27d9c8f6326e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/c8f4a223dc2d4e5986020ee4426f9b12, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/5e901652649543cabdd7f5e6027b7119, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/37aa650f17f34a259b34624b4709fb74, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/58d5c07a104440fbadd5fe385a7f9301, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/94849c30102544099c364402845a83f0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b8d82c67d9284869bda12076ae4ca862, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0688f052f062499280a2d6cff3bdcc8a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/9c619e0b56cf4bd08d72e634453e4ae7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/2fac7b2a623c42eea2987d70b2ff7dc6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/99b62ccabcbe49a8ba9742345772dd9a] to archive 2024-11-13T18:36:43,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:36:43,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/699ba810500941108d6fc925601c26d0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/699ba810500941108d6fc925601c26d0 2024-11-13T18:36:43,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa9734218de8459f82ada23061493d5f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa9734218de8459f82ada23061493d5f 2024-11-13T18:36:43,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/463914939a2c436581149a3ef5c00bf1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/463914939a2c436581149a3ef5c00bf1 2024-11-13T18:36:43,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/71857942fb5f483393f5f36bdd8619e2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/71857942fb5f483393f5f36bdd8619e2 2024-11-13T18:36:43,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa4b00398fc042fc878b63cd949c4ac9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/fa4b00398fc042fc878b63cd949c4ac9 2024-11-13T18:36:43,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/48e808bf306f498c85e2f204f661fda6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/48e808bf306f498c85e2f204f661fda6 2024-11-13T18:36:43,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/55535ec929bb43a8819a4fafaaabb258 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/55535ec929bb43a8819a4fafaaabb258 2024-11-13T18:36:43,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/285fa4c23a4d47a59136f6f7974dc55c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/285fa4c23a4d47a59136f6f7974dc55c 2024-11-13T18:36:43,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/947095edbca0457c9eef01b2e45f6059 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/947095edbca0457c9eef01b2e45f6059 2024-11-13T18:36:43,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/a2c1462d03a042c781f5d0bfb4270bbe to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/a2c1462d03a042c781f5d0bfb4270bbe 2024-11-13T18:36:43,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/1d08f51bdbe94fdb93f31e3618bbfd4a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/1d08f51bdbe94fdb93f31e3618bbfd4a 2024-11-13T18:36:43,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0bb983c92fac4742bb4431ce671c60d3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0bb983c92fac4742bb4431ce671c60d3 2024-11-13T18:36:43,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/440650511d5c4d6992940a6bb7e22525 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/440650511d5c4d6992940a6bb7e22525 2024-11-13T18:36:43,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0e5eca012bd0473e97a7bc56e3f4950f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0e5eca012bd0473e97a7bc56e3f4950f 2024-11-13T18:36:43,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/debdff204a78404f96d2b158e5217e6b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/debdff204a78404f96d2b158e5217e6b 2024-11-13T18:36:43,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ebf47cd69a324b28ac0c3342f5b3ebea to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ebf47cd69a324b28ac0c3342f5b3ebea 2024-11-13T18:36:43,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/e528f169f9354e58b94a27d9c8f6326e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/e528f169f9354e58b94a27d9c8f6326e 2024-11-13T18:36:43,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/c8f4a223dc2d4e5986020ee4426f9b12 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/c8f4a223dc2d4e5986020ee4426f9b12 2024-11-13T18:36:43,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/5e901652649543cabdd7f5e6027b7119 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/5e901652649543cabdd7f5e6027b7119 2024-11-13T18:36:43,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/37aa650f17f34a259b34624b4709fb74 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/37aa650f17f34a259b34624b4709fb74 2024-11-13T18:36:43,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/58d5c07a104440fbadd5fe385a7f9301 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/58d5c07a104440fbadd5fe385a7f9301 2024-11-13T18:36:43,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/94849c30102544099c364402845a83f0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/94849c30102544099c364402845a83f0 2024-11-13T18:36:43,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b8d82c67d9284869bda12076ae4ca862 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b8d82c67d9284869bda12076ae4ca862 2024-11-13T18:36:43,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0688f052f062499280a2d6cff3bdcc8a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/0688f052f062499280a2d6cff3bdcc8a 2024-11-13T18:36:43,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/9c619e0b56cf4bd08d72e634453e4ae7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/9c619e0b56cf4bd08d72e634453e4ae7 2024-11-13T18:36:43,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/2fac7b2a623c42eea2987d70b2ff7dc6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/2fac7b2a623c42eea2987d70b2ff7dc6 2024-11-13T18:36:43,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/99b62ccabcbe49a8ba9742345772dd9a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/99b62ccabcbe49a8ba9742345772dd9a 2024-11-13T18:36:43,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/af28e8b2f4ab4e7ea0b17a54c139baa5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/acc6e45ad5e8472d920b53a351155e21, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b4bd411c1cbe4592a32dcf247438f111, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ebb19d191f6a4230a260f9c88e1b550c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1d020c0b0f4a4a6689d19d495d4ffe63, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/046ce17ba1f24089ba8c5b5d755671f6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a0d1d959748047e4bedc104976c818b0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/88a80045c176482ba7fa3e868bec0b06, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/4234b0d919f04779870ff50d7b290b7b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b39c25489061412c9e75413f73e96c88, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c158e7f8465a4f149464c359ab51b929, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/53afe89d0115406cbe37a62207e96a88, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/94707c36f2ee4e09ab333037e9b5686c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/581735a1f1f54f96aef018c8f3173860, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/689a000d2cbe46e8b2bd8d815475a8b2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b9e4bc7d526a4388a533bab1e686d21e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/12eca725732f45ad9677be402e5cdd07, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7ba1f94fac7c45aebea91a3d6237e804, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/6767e74fd1e64cc1aeb9686532e40d87, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7b0fe3e846884d76979fc00aeb60f7b8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/fcfb10abfc424e33a00f263daa3fb88a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ad59f7798e8a48e2b277191e2e70935c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/32fbe7c9aefa46f8aad758ba92a6bdc5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a8fa39bd0dcf49458d67ddbe2898cdc2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/25e9b8ad703844ad863621af24bac97b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1c137a0c270b4081902d8c17b07ddc58] to archive 2024-11-13T18:36:43,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:36:43,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:36:43,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/af28e8b2f4ab4e7ea0b17a54c139baa5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/af28e8b2f4ab4e7ea0b17a54c139baa5 2024-11-13T18:36:43,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/acc6e45ad5e8472d920b53a351155e21 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/acc6e45ad5e8472d920b53a351155e21 2024-11-13T18:36:43,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b4bd411c1cbe4592a32dcf247438f111 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b4bd411c1cbe4592a32dcf247438f111 2024-11-13T18:36:43,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ebb19d191f6a4230a260f9c88e1b550c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ebb19d191f6a4230a260f9c88e1b550c 2024-11-13T18:36:43,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1d020c0b0f4a4a6689d19d495d4ffe63 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1d020c0b0f4a4a6689d19d495d4ffe63 2024-11-13T18:36:43,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/046ce17ba1f24089ba8c5b5d755671f6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/046ce17ba1f24089ba8c5b5d755671f6 2024-11-13T18:36:43,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a0d1d959748047e4bedc104976c818b0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a0d1d959748047e4bedc104976c818b0 2024-11-13T18:36:43,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/88a80045c176482ba7fa3e868bec0b06 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/88a80045c176482ba7fa3e868bec0b06 2024-11-13T18:36:43,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/4234b0d919f04779870ff50d7b290b7b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/4234b0d919f04779870ff50d7b290b7b 2024-11-13T18:36:43,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b39c25489061412c9e75413f73e96c88 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b39c25489061412c9e75413f73e96c88 2024-11-13T18:36:43,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c158e7f8465a4f149464c359ab51b929 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c158e7f8465a4f149464c359ab51b929 2024-11-13T18:36:43,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/53afe89d0115406cbe37a62207e96a88 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/53afe89d0115406cbe37a62207e96a88 2024-11-13T18:36:43,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/94707c36f2ee4e09ab333037e9b5686c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/94707c36f2ee4e09ab333037e9b5686c 2024-11-13T18:36:44,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/581735a1f1f54f96aef018c8f3173860 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/581735a1f1f54f96aef018c8f3173860 2024-11-13T18:36:44,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/689a000d2cbe46e8b2bd8d815475a8b2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/689a000d2cbe46e8b2bd8d815475a8b2 2024-11-13T18:36:44,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b9e4bc7d526a4388a533bab1e686d21e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/b9e4bc7d526a4388a533bab1e686d21e 2024-11-13T18:36:44,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/12eca725732f45ad9677be402e5cdd07 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/12eca725732f45ad9677be402e5cdd07 2024-11-13T18:36:44,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7ba1f94fac7c45aebea91a3d6237e804 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7ba1f94fac7c45aebea91a3d6237e804 2024-11-13T18:36:44,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/6767e74fd1e64cc1aeb9686532e40d87 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/6767e74fd1e64cc1aeb9686532e40d87 2024-11-13T18:36:44,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7b0fe3e846884d76979fc00aeb60f7b8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/7b0fe3e846884d76979fc00aeb60f7b8 2024-11-13T18:36:44,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/fcfb10abfc424e33a00f263daa3fb88a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/fcfb10abfc424e33a00f263daa3fb88a 2024-11-13T18:36:44,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ad59f7798e8a48e2b277191e2e70935c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ad59f7798e8a48e2b277191e2e70935c 2024-11-13T18:36:44,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/32fbe7c9aefa46f8aad758ba92a6bdc5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/32fbe7c9aefa46f8aad758ba92a6bdc5 2024-11-13T18:36:44,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/ee4d7c78ea5d4ac0b006a79a8b08f7f5 2024-11-13T18:36:44,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a8fa39bd0dcf49458d67ddbe2898cdc2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/a8fa39bd0dcf49458d67ddbe2898cdc2 2024-11-13T18:36:44,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/25e9b8ad703844ad863621af24bac97b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/25e9b8ad703844ad863621af24bac97b 2024-11-13T18:36:44,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1c137a0c270b4081902d8c17b07ddc58 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/1c137a0c270b4081902d8c17b07ddc58 2024-11-13T18:36:44,372 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=1 2024-11-13T18:36:44,373 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. 2024-11-13T18:36:44,373 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for 430b0529634d320a32ba155ab3165681: Waiting for close lock at 1731523001115Running coprocessor pre-close hooks at 1731523001115Disabling compacts and flushes for region at 1731523001115Disabling writes for close at 1731523001731 (+616 ms)Obtaining lock to block concurrent updates at 1731523001732 (+1 ms)Preparing flush snapshotting stores in 430b0529634d320a32ba155ab3165681 at 1731523001732Finished memstore snapshotting TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731523001732Flushing stores of TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681. at 1731523001735 (+3 ms)Flushing 430b0529634d320a32ba155ab3165681/A: creating writer at 1731523001735Flushing 430b0529634d320a32ba155ab3165681/A: appending metadata at 1731523001859 (+124 ms)Flushing 430b0529634d320a32ba155ab3165681/A: closing flushed file at 1731523001859Flushing 430b0529634d320a32ba155ab3165681/B: creating writer at 1731523002282 (+423 ms)Flushing 430b0529634d320a32ba155ab3165681/B: appending metadata at 1731523002303 (+21 ms)Flushing 430b0529634d320a32ba155ab3165681/B: closing flushed file at 1731523002303Flushing 430b0529634d320a32ba155ab3165681/C: creating writer at 1731523002524 (+221 ms)Flushing 430b0529634d320a32ba155ab3165681/C: appending metadata at 1731523002726 (+202 ms)Flushing 430b0529634d320a32ba155ab3165681/C: closing flushed file at 1731523002726Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5affd07f: reopening flushed file at 1731523002868 (+142 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c72aafd: reopening flushed file at 1731523002885 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cdc04fc: reopening flushed file at 1731523002901 (+16 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 430b0529634d320a32ba155ab3165681 in 1250ms, sequenceid=433, compaction requested=true at 1731523002981 (+80 ms)Writing region close event to WAL at 1731523004324 (+1343 ms)Running coprocessor post-close hooks at 1731523004373 (+49 ms)Closed at 1731523004373 2024-11-13T18:36:44,383 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=430b0529634d320a32ba155ab3165681, regionState=CLOSED 2024-11-13T18:36:44,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:36:44,386 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed 430b0529634d320a32ba155ab3165681 2024-11-13T18:36:44,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T18:36:44,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure 430b0529634d320a32ba155ab3165681, server=39e84130bbc9,35235,1731522975587 in 3.5030 sec 2024-11-13T18:36:44,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T18:36:44,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=430b0529634d320a32ba155ab3165681, UNASSIGN in 3.5410 sec 2024-11-13T18:36:44,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T18:36:44,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.7110 sec 2024-11-13T18:36:44,600 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523004600"}]},"ts":"1731523004600"} 2024-11-13T18:36:44,603 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:36:44,603 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:36:44,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.7400 sec 2024-11-13T18:36:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T18:36:45,044 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:45,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:36:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:45,070 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:45,073 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:36:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:36:45,208 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681 2024-11-13T18:36:45,293 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:36:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:36:45,423 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/recovered.edits] 2024-11-13T18:36:45,431 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/64ef5ad875574a81b1d23ccd71937c55 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/64ef5ad875574a81b1d23ccd71937c55 2024-11-13T18:36:45,434 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/949a680cfb134bf4966825aa6e84794a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/949a680cfb134bf4966825aa6e84794a 2024-11-13T18:36:45,513 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ae7d26c20eb64b1baf2e279eab0c1642 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/ae7d26c20eb64b1baf2e279eab0c1642 2024-11-13T18:36:45,555 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/c27b7cfed1d949e7aacff58becac9891 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/A/c27b7cfed1d949e7aacff58becac9891 2024-11-13T18:36:45,559 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b07b52905df8410daebd3e2f79e051f5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/b07b52905df8410daebd3e2f79e051f5 2024-11-13T18:36:45,564 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/dee93140d64541af9d8f59b4e8e50fc9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/dee93140d64541af9d8f59b4e8e50fc9 2024-11-13T18:36:45,568 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/e0bd5adfe4ec454ca178a314c3fe83e7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/e0bd5adfe4ec454ca178a314c3fe83e7 2024-11-13T18:36:45,571 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ef3018f15e4140e8b194ed55102d2196 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/B/ef3018f15e4140e8b194ed55102d2196 2024-11-13T18:36:45,575 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/80c00858d793456ea8a8520c68ca2210 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/80c00858d793456ea8a8520c68ca2210 2024-11-13T18:36:45,579 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/87c2126c107e41b89b80a0749c46c18d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/87c2126c107e41b89b80a0749c46c18d 2024-11-13T18:36:45,581 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c48cfd99f9c249fa97c793b2f104ca9b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/c48cfd99f9c249fa97c793b2f104ca9b 2024-11-13T18:36:45,583 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/d11f197e23524d3a9acd12bcd730b284 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/C/d11f197e23524d3a9acd12bcd730b284 2024-11-13T18:36:45,595 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/recovered.edits/436.seqid to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681/recovered.edits/436.seqid 2024-11-13T18:36:45,596 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/430b0529634d320a32ba155ab3165681 2024-11-13T18:36:45,596 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:36:45,607 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:45,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T18:36:45,660 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:36:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:36:45,966 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:36:46,048 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:46,049 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:36:46,139 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731523006049"}]},"ts":"9223372036854775807"} 2024-11-13T18:36:46,146 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:36:46,146 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 430b0529634d320a32ba155ab3165681, NAME => 'TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:36:46,146 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:36:46,147 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731523006146"}]},"ts":"9223372036854775807"} 2024-11-13T18:36:46,157 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:36:46,159 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:46,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 1.0970 sec 2024-11-13T18:36:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T18:36:46,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:36:46,245 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:46,302 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=240 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;39e84130bbc9:35235-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314521622_22 at /127.0.0.1:49958 [Waiting for operation #48] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1368938180_22 at /127.0.0.1:40710 [Waiting for operation #203] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1368938180_22 at /127.0.0.1:40876 [Waiting for operation #188] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314521622_22 at /127.0.0.1:49566 [Waiting for operation #152] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=2312 (was 446) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3205 (was 3065) - AvailableMemoryMB LEAK? - 2024-11-13T18:36:46,511 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=2312, ProcessCount=11, AvailableMemoryMB=3204 2024-11-13T18:36:46,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:36:46,515 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:36:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:46,521 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:36:46,521 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:46,522 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:36:46,522 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-13T18:36:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:36:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:36:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:36:46,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741930_1106 (size=959) 2024-11-13T18:36:46,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 2024-11-13T18:36:46,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741931_1107 (size=53) 2024-11-13T18:36:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:36:47,398 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:47,398 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4f01271d10fc9cb8342007ba2bfde68b, disabling compactions & flushes 2024-11-13T18:36:47,398 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:47,398 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:47,398 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. after waiting 0 ms 2024-11-13T18:36:47,398 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:47,398 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:47,398 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4f01271d10fc9cb8342007ba2bfde68b: Waiting for close lock at 1731523007398Disabling compacts and flushes for region at 1731523007398Disabling writes for close at 1731523007398Writing region close event to WAL at 1731523007398Closed at 1731523007398 2024-11-13T18:36:47,401 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:36:47,439 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731523007401"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731523007401"}]},"ts":"1731523007401"} 2024-11-13T18:36:47,442 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:36:47,444 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:36:47,448 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523007445"}]},"ts":"1731523007445"} 2024-11-13T18:36:47,462 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:36:47,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, ASSIGN}] 2024-11-13T18:36:47,469 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, ASSIGN 2024-11-13T18:36:47,470 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, ASSIGN; state=OFFLINE, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=false 2024-11-13T18:36:47,626 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=4f01271d10fc9cb8342007ba2bfde68b, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:47,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, ASSIGN because future has completed 2024-11-13T18:36:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:36:47,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:36:47,966 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:47,966 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:36:47,967 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:47,967 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:47,967 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:47,967 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,101 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,143 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f01271d10fc9cb8342007ba2bfde68b columnFamilyName A 2024-11-13T18:36:48,143 DEBUG [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:48,144 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(327): Store=4f01271d10fc9cb8342007ba2bfde68b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:48,144 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,145 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f01271d10fc9cb8342007ba2bfde68b columnFamilyName B 2024-11-13T18:36:48,145 DEBUG [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:48,148 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(327): Store=4f01271d10fc9cb8342007ba2bfde68b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:48,148 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,213 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f01271d10fc9cb8342007ba2bfde68b columnFamilyName C 2024-11-13T18:36:48,213 DEBUG [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:48,221 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(327): Store=4f01271d10fc9cb8342007ba2bfde68b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:48,221 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,230 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,231 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,233 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,233 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,233 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:36:48,251 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,404 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:36:48,405 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 4f01271d10fc9cb8342007ba2bfde68b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62125200, jitterRate=-0.07426238059997559}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:36:48,405 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:48,406 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 4f01271d10fc9cb8342007ba2bfde68b: Running coprocessor pre-open hook at 1731523007967Writing region info on filesystem at 1731523007967Initializing all the Stores at 1731523007970 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523007970Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523008100 (+130 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523008100Cleaning up temporary data from old regions at 1731523008233 (+133 ms)Running coprocessor post-open hooks at 1731523008405 (+172 ms)Region opened successfully at 1731523008406 (+1 ms) 2024-11-13T18:36:48,407 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., pid=26, masterSystemTime=1731523007858 2024-11-13T18:36:48,410 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:48,411 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:48,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=4f01271d10fc9cb8342007ba2bfde68b, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:48,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:36:48,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T18:36:48,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 in 734 msec 2024-11-13T18:36:48,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-13T18:36:48,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, ASSIGN in 984 msec 2024-11-13T18:36:48,464 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:36:48,464 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523008464"}]},"ts":"1731523008464"} 2024-11-13T18:36:48,466 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:36:48,474 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:36:48,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.9590 sec 2024-11-13T18:36:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T18:36:48,674 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:48,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:48,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:48,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:48,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:48,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2abf9778, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:48,770 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e803881, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:48,772 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:48,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:48,777 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:36:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 39e84130bbc9,36105,1731522974847 2024-11-13T18:36:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c8881d2 2024-11-13T18:36:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:36:48,780 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:36:48,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:36:48,789 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:36:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:36:49,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741932_1108 (size=995) 2024-11-13T18:36:49,084 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T18:36:49,084 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T18:36:49,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:36:49,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, REOPEN/MOVE}] 2024-11-13T18:36:49,124 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, REOPEN/MOVE 2024-11-13T18:36:49,126 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4f01271d10fc9cb8342007ba2bfde68b, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:49,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, REOPEN/MOVE because future has completed 2024-11-13T18:36:49,128 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:36:49,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:36:49,289 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,289 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:36:49,289 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 4f01271d10fc9cb8342007ba2bfde68b, disabling compactions & flushes 2024-11-13T18:36:49,290 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:49,290 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:49,290 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. after waiting 0 ms 2024-11-13T18:36:49,290 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:49,359 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:36:49,360 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:49,360 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 4f01271d10fc9cb8342007ba2bfde68b: Waiting for close lock at 1731523009289Running coprocessor pre-close hooks at 1731523009289Disabling compacts and flushes for region at 1731523009289Disabling writes for close at 1731523009290 (+1 ms)Writing region close event to WAL at 1731523009331 (+41 ms)Running coprocessor post-close hooks at 1731523009360 (+29 ms)Closed at 1731523009360 2024-11-13T18:36:49,361 WARN [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: 4f01271d10fc9cb8342007ba2bfde68b to self. 2024-11-13T18:36:49,364 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,364 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4f01271d10fc9cb8342007ba2bfde68b, regionState=CLOSED 2024-11-13T18:36:49,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:36:49,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T18:36:49,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 in 250 msec 2024-11-13T18:36:49,381 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, REOPEN/MOVE; state=CLOSED, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=true 2024-11-13T18:36:49,532 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4f01271d10fc9cb8342007ba2bfde68b, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:49,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, REOPEN/MOVE because future has completed 2024-11-13T18:36:49,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:36:49,706 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:49,706 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:36:49,707 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,707 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:36:49,707 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,707 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,838 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,856 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f01271d10fc9cb8342007ba2bfde68b columnFamilyName A 2024-11-13T18:36:49,949 DEBUG [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:49,958 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(327): Store=4f01271d10fc9cb8342007ba2bfde68b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:49,958 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,960 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f01271d10fc9cb8342007ba2bfde68b columnFamilyName B 2024-11-13T18:36:49,960 DEBUG [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:49,961 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(327): Store=4f01271d10fc9cb8342007ba2bfde68b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:49,961 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,981 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f01271d10fc9cb8342007ba2bfde68b columnFamilyName C 2024-11-13T18:36:49,981 DEBUG [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:49,982 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:36:49,982 INFO [StoreOpener-4f01271d10fc9cb8342007ba2bfde68b-1 {}] regionserver.HStore(327): Store=4f01271d10fc9cb8342007ba2bfde68b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:36:49,982 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,984 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:49,998 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:50,030 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:50,030 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:50,031 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:36:50,033 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:50,034 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 4f01271d10fc9cb8342007ba2bfde68b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63735385, jitterRate=-0.05026875436306}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:36:50,034 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:50,035 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 4f01271d10fc9cb8342007ba2bfde68b: Running coprocessor pre-open hook at 1731523009707Writing region info on filesystem at 1731523009707Initializing all the Stores at 1731523009708 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523009708Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523009773 (+65 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523009773Cleaning up temporary data from old regions at 1731523010030 (+257 ms)Running coprocessor post-open hooks at 1731523010034 (+4 ms)Region opened successfully at 1731523010035 (+1 ms) 2024-11-13T18:36:50,036 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., pid=31, masterSystemTime=1731523009698 2024-11-13T18:36:50,038 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:50,038 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:50,039 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4f01271d10fc9cb8342007ba2bfde68b, regionState=OPEN, openSeqNum=5, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:50,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:36:50,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-13T18:36:50,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 in 498 msec 2024-11-13T18:36:50,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T18:36:50,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, REOPEN/MOVE in 923 msec 2024-11-13T18:36:50,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T18:36:50,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 962 msec 2024-11-13T18:36:50,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.2650 sec 2024-11-13T18:36:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T18:36:50,061 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:50,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,142 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c44d68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,145 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4f64ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,167 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c07527f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,169 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc6dd64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,253 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae54cf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,256 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a920c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,285 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aff69c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@434ac1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d1ada53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db2ddf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,350 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@397b2b09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,369 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c61590, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,391 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2491e498, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,394 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7779e6ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,512 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9d50a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,515 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0f0c5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:36:50,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:36:50,585 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:36:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:36:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:36:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e70375c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:36:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:36:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:36:50,589 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:36:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1a2f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:36:50,593 DEBUG [Thread-572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,595 DEBUG [Thread-572 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,635 DEBUG [Thread-574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,647 DEBUG [Thread-583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,663 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,678 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:50,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:36:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-13T18:36:50,681 DEBUG [Thread-579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,681 DEBUG [Thread-577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,682 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:50,682 DEBUG [Thread-566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:36:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,684 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:50,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:50,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,686 DEBUG [Thread-568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:36:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:50,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:50,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:36:50,883 DEBUG [Thread-570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,910 DEBUG [Thread-581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:36:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:36:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:36:50,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:36:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:36:50,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523070966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:50,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523070973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:50,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:50,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:50,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523070983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523070986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523070987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:36:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:36:51,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523071097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523071103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523071114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523071114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523071115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:36:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113374dea615e6f44fbb9b672be86561ea5_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523010674/Put/seqid=0 2024-11-13T18:36:51,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741933_1109 (size=19474) 2024-11-13T18:36:51,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:51,345 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113374dea615e6f44fbb9b672be86561ea5_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113374dea615e6f44fbb9b672be86561ea5_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:51,347 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/9bc9d1980d2047aab2cb76e20740e146, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:51,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/9bc9d1980d2047aab2cb76e20740e146 is 175, key is test_row_0/A:col10/1731523010674/Put/seqid=0 2024-11-13T18:36:51,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:36:51,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:51,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:36:51,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523071418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523071418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523071418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741934_1110 (size=56733) 2024-11-13T18:36:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523071418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523071419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,443 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/9bc9d1980d2047aab2cb76e20740e146 2024-11-13T18:36:51,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/21bc13ce5be84398b159d7a628cce65e is 50, key is test_row_0/B:col10/1731523010674/Put/seqid=0 2024-11-13T18:36:51,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:36:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741935_1111 (size=12001) 2024-11-13T18:36:51,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/21bc13ce5be84398b159d7a628cce65e 2024-11-13T18:36:51,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/e4c1fb41a9024869bf58aa4fb1e1eddb is 50, key is test_row_0/C:col10/1731523010674/Put/seqid=0 2024-11-13T18:36:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523071743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523071744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523071744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523071744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:36:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523071753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:36:51,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:36:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:51,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:52,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741936_1112 (size=12001) 2024-11-13T18:36:52,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/e4c1fb41a9024869bf58aa4fb1e1eddb 2024-11-13T18:36:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/9bc9d1980d2047aab2cb76e20740e146 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9bc9d1980d2047aab2cb76e20740e146 2024-11-13T18:36:52,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9bc9d1980d2047aab2cb76e20740e146, entries=300, sequenceid=15, filesize=55.4 K 2024-11-13T18:36:52,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/21bc13ce5be84398b159d7a628cce65e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/21bc13ce5be84398b159d7a628cce65e 2024-11-13T18:36:52,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/21bc13ce5be84398b159d7a628cce65e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T18:36:52,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/e4c1fb41a9024869bf58aa4fb1e1eddb as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e4c1fb41a9024869bf58aa4fb1e1eddb 2024-11-13T18:36:52,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e4c1fb41a9024869bf58aa4fb1e1eddb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T18:36:52,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4f01271d10fc9cb8342007ba2bfde68b in 1505ms, sequenceid=15, compaction requested=false 2024-11-13T18:36:52,184 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T18:36:52,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:52,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T18:36:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:52,223 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:36:52,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:52,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111369be22121eeb46639f0c791ae9a81018_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523010928/Put/seqid=0 2024-11-13T18:36:52,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523072330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523072384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523072391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523072394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523072401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741937_1113 (size=12154) 2024-11-13T18:36:52,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:52,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523072518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523072597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523072597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523072603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523072603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,693 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111369be22121eeb46639f0c791ae9a81018_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111369be22121eeb46639f0c791ae9a81018_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:52,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/614db059e5794df48e56666e856a8dce, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:52,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/614db059e5794df48e56666e856a8dce is 175, key is test_row_0/A:col10/1731523010928/Put/seqid=0 2024-11-13T18:36:52,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741938_1114 (size=30955) 2024-11-13T18:36:52,891 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/614db059e5794df48e56666e856a8dce 2024-11-13T18:36:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:36:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523072878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:52,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523072878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523072879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523072879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:53,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523072894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/a831a8d1ef2446ec9994ddf72cada0ea is 50, key is test_row_0/B:col10/1731523010928/Put/seqid=0 2024-11-13T18:36:53,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741939_1115 (size=12001) 2024-11-13T18:36:53,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:53,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523073284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523073294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523073413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523073413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523073417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:53,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:53,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:53,712 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/a831a8d1ef2446ec9994ddf72cada0ea 2024-11-13T18:36:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/390948f484994adda1b254165e6a7370 is 50, key is test_row_0/C:col10/1731523010928/Put/seqid=0 2024-11-13T18:36:53,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741940_1116 (size=12001) 2024-11-13T18:36:53,857 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/390948f484994adda1b254165e6a7370 2024-11-13T18:36:53,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/614db059e5794df48e56666e856a8dce as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/614db059e5794df48e56666e856a8dce 2024-11-13T18:36:53,875 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/614db059e5794df48e56666e856a8dce, entries=150, sequenceid=40, filesize=30.2 K 2024-11-13T18:36:53,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/a831a8d1ef2446ec9994ddf72cada0ea as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/a831a8d1ef2446ec9994ddf72cada0ea 2024-11-13T18:36:53,886 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/a831a8d1ef2446ec9994ddf72cada0ea, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T18:36:53,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/390948f484994adda1b254165e6a7370 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/390948f484994adda1b254165e6a7370 2024-11-13T18:36:53,911 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/390948f484994adda1b254165e6a7370, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T18:36:53,912 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4f01271d10fc9cb8342007ba2bfde68b in 1689ms, sequenceid=40, compaction requested=false 2024-11-13T18:36:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-13T18:36:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-13T18:36:53,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T18:36:53,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2310 sec 2024-11-13T18:36:53,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 3.2560 sec 2024-11-13T18:36:54,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:54,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:36:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:54,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132bedc91df1e54bb5bbc1e45181aeabe2_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523012310/Put/seqid=0 2024-11-13T18:36:54,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741941_1117 (size=14594) 2024-11-13T18:36:54,522 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:54,530 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132bedc91df1e54bb5bbc1e45181aeabe2_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132bedc91df1e54bb5bbc1e45181aeabe2_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:54,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/87f4e1ea471e4b6385ceb46a4f51afad, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:54,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/87f4e1ea471e4b6385ceb46a4f51afad is 175, key is test_row_0/A:col10/1731523012310/Put/seqid=0 2024-11-13T18:36:54,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741942_1118 (size=39549) 2024-11-13T18:36:54,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/87f4e1ea471e4b6385ceb46a4f51afad 2024-11-13T18:36:54,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/12e39c83cc5440b98c44ce10caf60559 is 50, key is test_row_0/B:col10/1731523012310/Put/seqid=0 2024-11-13T18:36:54,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523074609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:54,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523074613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:54,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741943_1119 (size=12001) 2024-11-13T18:36:54,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/12e39c83cc5440b98c44ce10caf60559 2024-11-13T18:36:54,863 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:36:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T18:36:54,974 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:54,975 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-13T18:36:54,978 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:54,979 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:54,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:36:54,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523074659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523074755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523074769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523074863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523074876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/1f4d4213fe2544e6accccd57c9af4f18 is 50, key is test_row_0/C:col10/1731523012310/Put/seqid=0 2024-11-13T18:36:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:36:55,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:36:55,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:55,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:36:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:36:55,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523075104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:55,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523075104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523075163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523075264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523075264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:55,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:36:55,372 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T18:36:55,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741944_1120 (size=12001) 2024-11-13T18:36:55,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/1f4d4213fe2544e6accccd57c9af4f18 2024-11-13T18:36:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/87f4e1ea471e4b6385ceb46a4f51afad as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/87f4e1ea471e4b6385ceb46a4f51afad 2024-11-13T18:36:55,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/87f4e1ea471e4b6385ceb46a4f51afad, entries=200, sequenceid=52, filesize=38.6 K 2024-11-13T18:36:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/12e39c83cc5440b98c44ce10caf60559 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/12e39c83cc5440b98c44ce10caf60559 2024-11-13T18:36:55,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/12e39c83cc5440b98c44ce10caf60559, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T18:36:55,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/1f4d4213fe2544e6accccd57c9af4f18 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1f4d4213fe2544e6accccd57c9af4f18 2024-11-13T18:36:55,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1f4d4213fe2544e6accccd57c9af4f18, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T18:36:55,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:36:55,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:55,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 4f01271d10fc9cb8342007ba2bfde68b in 1354ms, sequenceid=52, compaction requested=true 2024-11-13T18:36:55,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:55,445 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:55,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:55,446 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:55,446 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/A is initiating minor compaction (all files) 2024-11-13T18:36:55,446 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/A in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,446 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9bc9d1980d2047aab2cb76e20740e146, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/614db059e5794df48e56666e856a8dce, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/87f4e1ea471e4b6385ceb46a4f51afad] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=124.3 K 2024-11-13T18:36:55,447 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,447 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9bc9d1980d2047aab2cb76e20740e146, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/614db059e5794df48e56666e856a8dce, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/87f4e1ea471e4b6385ceb46a4f51afad] 2024-11-13T18:36:55,448 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bc9d1980d2047aab2cb76e20740e146, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731523010606 2024-11-13T18:36:55,448 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 614db059e5794df48e56666e856a8dce, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731523010928 2024-11-13T18:36:55,449 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87f4e1ea471e4b6385ceb46a4f51afad, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731523012310 2024-11-13T18:36:55,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:55,450 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:55,452 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:55,452 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/B is initiating minor compaction (all files) 2024-11-13T18:36:55,452 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/B in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,452 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/21bc13ce5be84398b159d7a628cce65e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/a831a8d1ef2446ec9994ddf72cada0ea, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/12e39c83cc5440b98c44ce10caf60559] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=35.2 K 2024-11-13T18:36:55,454 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bc13ce5be84398b159d7a628cce65e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731523010662 2024-11-13T18:36:55,455 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a831a8d1ef2446ec9994ddf72cada0ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731523010928 2024-11-13T18:36:55,456 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e39c83cc5440b98c44ce10caf60559, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731523012310 2024-11-13T18:36:55,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:55,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:55,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:55,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:55,516 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#B#compaction#102 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:55,519 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/3162e0b683aa45f4a49ae480f9f0071d is 50, key is test_row_0/B:col10/1731523012310/Put/seqid=0 2024-11-13T18:36:55,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T18:36:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:55,598 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:36:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:36:55,674 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:55,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:55,792 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113fea78ee544624d1392421bf9ca2fcd98_4f01271d10fc9cb8342007ba2bfde68b store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:55,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741945_1121 (size=12104) 2024-11-13T18:36:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113979c7fe94f184d82b83c4feed0724860_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523014718/Put/seqid=0 2024-11-13T18:36:56,009 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/3162e0b683aa45f4a49ae480f9f0071d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3162e0b683aa45f4a49ae480f9f0071d 2024-11-13T18:36:56,071 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113fea78ee544624d1392421bf9ca2fcd98_4f01271d10fc9cb8342007ba2bfde68b, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:56,071 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fea78ee544624d1392421bf9ca2fcd98_4f01271d10fc9cb8342007ba2bfde68b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:36:56,144 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/B of 4f01271d10fc9cb8342007ba2bfde68b into 3162e0b683aa45f4a49ae480f9f0071d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:56,144 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:56,144 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/B, priority=13, startTime=1731523015450; duration=0sec 2024-11-13T18:36:56,144 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:56,144 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:B 2024-11-13T18:36:56,144 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:56,146 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:56,147 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/C is initiating minor compaction (all files) 2024-11-13T18:36:56,147 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/C in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:56,147 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e4c1fb41a9024869bf58aa4fb1e1eddb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/390948f484994adda1b254165e6a7370, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1f4d4213fe2544e6accccd57c9af4f18] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=35.2 K 2024-11-13T18:36:56,147 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c1fb41a9024869bf58aa4fb1e1eddb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731523010662 2024-11-13T18:36:56,148 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 390948f484994adda1b254165e6a7370, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731523010928 2024-11-13T18:36:56,148 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4d4213fe2544e6accccd57c9af4f18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731523012310 2024-11-13T18:36:56,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741946_1122 (size=12154) 2024-11-13T18:36:56,414 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#C#compaction#105 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:56,415 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/e74c1194f59d49a09efd234aaec117c1 is 50, key is test_row_0/C:col10/1731523012310/Put/seqid=0 2024-11-13T18:36:56,431 INFO [AsyncFSWAL-0-hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462-prefix:39e84130bbc9,35235,1731522975587 {}] wal.AbstractFSWAL(1368): Slow sync cost: 167 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42373,DS-43627c6c-1ac2-4d78-b60b-b6c4203a554c,DISK]] 2024-11-13T18:36:56,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523075795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523075797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741947_1123 (size=4469) 2024-11-13T18:36:56,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:56,603 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113979c7fe94f184d82b83c4feed0724860_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113979c7fe94f184d82b83c4feed0724860_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/b9b6d1ba592b4095a4c3abc79916ae76, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:56,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/b9b6d1ba592b4095a4c3abc79916ae76 is 175, key is test_row_0/A:col10/1731523014718/Put/seqid=0 2024-11-13T18:36:56,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741948_1124 (size=12104) 2024-11-13T18:36:56,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523076434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523076448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523076470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,709 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/e74c1194f59d49a09efd234aaec117c1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e74c1194f59d49a09efd234aaec117c1 2024-11-13T18:36:56,720 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/C of 4f01271d10fc9cb8342007ba2bfde68b into e74c1194f59d49a09efd234aaec117c1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:56,720 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:56,720 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/C, priority=13, startTime=1731523015456; duration=0sec 2024-11-13T18:36:56,720 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:56,720 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:C 2024-11-13T18:36:56,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741949_1125 (size=30955) 2024-11-13T18:36:56,745 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/b9b6d1ba592b4095a4c3abc79916ae76 2024-11-13T18:36:56,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/d73528422e904b348833d6c99283f5b2 is 50, key is test_row_0/B:col10/1731523014718/Put/seqid=0 2024-11-13T18:36:56,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523076554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523076554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,879 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#A#compaction#103 average throughput is 0.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:56,880 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/53a7fba66caa4b69a280fd4f65163a71 is 175, key is test_row_0/A:col10/1731523012310/Put/seqid=0 2024-11-13T18:36:56,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741950_1126 (size=12001) 2024-11-13T18:36:56,885 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/d73528422e904b348833d6c99283f5b2 2024-11-13T18:36:56,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523076815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523076815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523076815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/182a7b2344df480bad70b8f9ba3aa75a is 50, key is test_row_0/C:col10/1731523014718/Put/seqid=0 2024-11-13T18:36:57,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741951_1127 (size=31058) 2024-11-13T18:36:57,069 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/53a7fba66caa4b69a280fd4f65163a71 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/53a7fba66caa4b69a280fd4f65163a71 2024-11-13T18:36:57,079 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/A of 4f01271d10fc9cb8342007ba2bfde68b into 53a7fba66caa4b69a280fd4f65163a71(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-13T18:36:57,079 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:57,079 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/A, priority=13, startTime=1731523015444; duration=1sec 2024-11-13T18:36:57,080 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:57,080 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:A 2024-11-13T18:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:36:57,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523077054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523077104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523077104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523077104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523077054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741952_1128 (size=12001) 2024-11-13T18:36:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523077498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523077504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523077504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523077505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523077505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:57,600 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/182a7b2344df480bad70b8f9ba3aa75a 2024-11-13T18:36:57,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/b9b6d1ba592b4095a4c3abc79916ae76 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/b9b6d1ba592b4095a4c3abc79916ae76 2024-11-13T18:36:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:57,654 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/b9b6d1ba592b4095a4c3abc79916ae76, entries=150, sequenceid=77, filesize=30.2 K 2024-11-13T18:36:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/d73528422e904b348833d6c99283f5b2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/d73528422e904b348833d6c99283f5b2 2024-11-13T18:36:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,662 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/d73528422e904b348833d6c99283f5b2, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T18:36:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/182a7b2344df480bad70b8f9ba3aa75a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/182a7b2344df480bad70b8f9ba3aa75a 2024-11-13T18:36:57,714 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/182a7b2344df480bad70b8f9ba3aa75a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T18:36:57,716 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4f01271d10fc9cb8342007ba2bfde68b in 2117ms, sequenceid=77, compaction requested=false 2024-11-13T18:36:57,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:57,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-13T18:36:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-13T18:36:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-13T18:36:57,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7720 sec 2024-11-13T18:36:57,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.7810 sec 2024-11-13T18:36:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:58,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:36:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,249 INFO [master/39e84130bbc9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T18:36:58,249 INFO [master/39e84130bbc9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T18:36:58,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137cf342aac31448fbbb24d7e105beb42e_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523016435/Put/seqid=0 2024-11-13T18:36:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741953_1129 (size=12154) 2024-11-13T18:36:58,378 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,387 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137cf342aac31448fbbb24d7e105beb42e_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137cf342aac31448fbbb24d7e105beb42e_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:58,388 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/5f77d7637d1749c2b62d89dfa2d3ae96, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:58,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/5f77d7637d1749c2b62d89dfa2d3ae96 is 175, key is test_row_0/A:col10/1731523016435/Put/seqid=0 2024-11-13T18:36:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741954_1130 (size=30951) 2024-11-13T18:36:58,411 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/5f77d7637d1749c2b62d89dfa2d3ae96 2024-11-13T18:36:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/17e6d70d4713453dbad99f9f2e7f027b is 50, key is test_row_0/B:col10/1731523016435/Put/seqid=0 2024-11-13T18:36:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741955_1131 (size=9657) 2024-11-13T18:36:58,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/17e6d70d4713453dbad99f9f2e7f027b 2024-11-13T18:36:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/7c6ca5289bc5466e9c8a78e8ffc90018 is 50, key is test_row_0/C:col10/1731523016435/Put/seqid=0 2024-11-13T18:36:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:58,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523078864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523078866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523078871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523078894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523078903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:58,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741956_1132 (size=9657) 2024-11-13T18:36:58,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/7c6ca5289bc5466e9c8a78e8ffc90018 2024-11-13T18:36:58,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/5f77d7637d1749c2b62d89dfa2d3ae96 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/5f77d7637d1749c2b62d89dfa2d3ae96 2024-11-13T18:36:58,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/5f77d7637d1749c2b62d89dfa2d3ae96, entries=150, sequenceid=92, filesize=30.2 K 2024-11-13T18:36:58,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/17e6d70d4713453dbad99f9f2e7f027b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/17e6d70d4713453dbad99f9f2e7f027b 2024-11-13T18:36:59,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/17e6d70d4713453dbad99f9f2e7f027b, entries=100, sequenceid=92, filesize=9.4 K 2024-11-13T18:36:59,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/7c6ca5289bc5466e9c8a78e8ffc90018 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/7c6ca5289bc5466e9c8a78e8ffc90018 2024-11-13T18:36:59,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/7c6ca5289bc5466e9c8a78e8ffc90018, entries=100, sequenceid=92, filesize=9.4 K 2024-11-13T18:36:59,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4f01271d10fc9cb8342007ba2bfde68b in 826ms, sequenceid=92, compaction requested=true 2024-11-13T18:36:59,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:59,015 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:59,017 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:59,017 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/A is initiating minor compaction (all files) 2024-11-13T18:36:59,017 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/A in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,017 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/53a7fba66caa4b69a280fd4f65163a71, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/b9b6d1ba592b4095a4c3abc79916ae76, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/5f77d7637d1749c2b62d89dfa2d3ae96] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=90.8 K 2024-11-13T18:36:59,017 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,017 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/53a7fba66caa4b69a280fd4f65163a71, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/b9b6d1ba592b4095a4c3abc79916ae76, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/5f77d7637d1749c2b62d89dfa2d3ae96] 2024-11-13T18:36:59,018 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 53a7fba66caa4b69a280fd4f65163a71, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731523012310 2024-11-13T18:36:59,018 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b9b6d1ba592b4095a4c3abc79916ae76, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731523014603 2024-11-13T18:36:59,019 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f77d7637d1749c2b62d89dfa2d3ae96, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523016425 2024-11-13T18:36:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:36:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:36:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:36:59,027 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:59,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:59,028 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:59,028 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/B is initiating minor compaction (all files) 2024-11-13T18:36:59,029 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/B in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,029 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3162e0b683aa45f4a49ae480f9f0071d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/d73528422e904b348833d6c99283f5b2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/17e6d70d4713453dbad99f9f2e7f027b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=33.0 K 2024-11-13T18:36:59,029 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3162e0b683aa45f4a49ae480f9f0071d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731523012310 2024-11-13T18:36:59,030 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:59,030 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d73528422e904b348833d6c99283f5b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731523014603 2024-11-13T18:36:59,030 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17e6d70d4713453dbad99f9f2e7f027b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523016435 2024-11-13T18:36:59,099 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411139bb9db6e60dc4971a2f772b3ba9e7b4d_4f01271d10fc9cb8342007ba2bfde68b store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:59,104 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411139bb9db6e60dc4971a2f772b3ba9e7b4d_4f01271d10fc9cb8342007ba2bfde68b, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:59,104 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139bb9db6e60dc4971a2f772b3ba9e7b4d_4f01271d10fc9cb8342007ba2bfde68b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:59,113 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#B#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:59,114 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/fc41bd5fc72f4e55ae5e2db40c78ede0 is 50, key is test_row_0/B:col10/1731523016435/Put/seqid=0 2024-11-13T18:36:59,145 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:36:59,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:36:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T18:36:59,149 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:36:59,150 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:36:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-13T18:36:59,152 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:36:59,153 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:36:59,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:36:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:36:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523079201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523079209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523079209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523079219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523079238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:36:59,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741957_1133 (size=4469) 2024-11-13T18:36:59,298 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#A#compaction#111 average throughput is 0.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:59,299 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/11bbb39504f84e82ad07090676ef732e is 175, key is test_row_0/A:col10/1731523016435/Put/seqid=0 2024-11-13T18:36:59,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:36:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741958_1134 (size=12207) 2024-11-13T18:36:59,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111347215677a9694327a41dce5f48191f71_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523018893/Put/seqid=0 2024-11-13T18:36:59,392 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/fc41bd5fc72f4e55ae5e2db40c78ede0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fc41bd5fc72f4e55ae5e2db40c78ede0 2024-11-13T18:36:59,406 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/B of 4f01271d10fc9cb8342007ba2bfde68b into fc41bd5fc72f4e55ae5e2db40c78ede0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:59,406 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:59,406 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/B, priority=13, startTime=1731523019027; duration=0sec 2024-11-13T18:36:59,407 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:36:59,407 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:B 2024-11-13T18:36:59,407 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:36:59,414 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:36:59,414 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/C is initiating minor compaction (all files) 2024-11-13T18:36:59,414 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/C in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,415 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e74c1194f59d49a09efd234aaec117c1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/182a7b2344df480bad70b8f9ba3aa75a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/7c6ca5289bc5466e9c8a78e8ffc90018] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=33.0 K 2024-11-13T18:36:59,415 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e74c1194f59d49a09efd234aaec117c1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731523012310 2024-11-13T18:36:59,421 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 182a7b2344df480bad70b8f9ba3aa75a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731523014603 2024-11-13T18:36:59,423 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6ca5289bc5466e9c8a78e8ffc90018, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523016435 2024-11-13T18:36:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:36:59,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:36:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523079359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523079364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523079365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523079374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523079364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,630 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:36:59,631 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/6eea27e7601f46f9af4690b283cd8d22 is 50, key is test_row_0/C:col10/1731523016435/Put/seqid=0 2024-11-13T18:36:59,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:36:59,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:59,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741959_1135 (size=31268) 2024-11-13T18:36:59,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523079699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741960_1136 (size=12154) 2024-11-13T18:36:59,737 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:36:59,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111347215677a9694327a41dce5f48191f71_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347215677a9694327a41dce5f48191f71_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:36:59,755 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/3463b0d024fd4260a61abcd5b3299a24, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:36:59,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/3463b0d024fd4260a61abcd5b3299a24 is 175, key is test_row_0/A:col10/1731523018893/Put/seqid=0 2024-11-13T18:36:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523079704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,757 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/11bbb39504f84e82ad07090676ef732e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/11bbb39504f84e82ad07090676ef732e 2024-11-13T18:36:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523079707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523079704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,776 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/A of 4f01271d10fc9cb8342007ba2bfde68b into 11bbb39504f84e82ad07090676ef732e(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:59,776 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:59,776 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/A, priority=13, startTime=1731523019015; duration=0sec 2024-11-13T18:36:59,777 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:59,777 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:A 2024-11-13T18:36:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:36:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523079749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:36:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:36:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:36:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:36:59,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:36:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:36:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:36:59,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:36:59,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741961_1137 (size=12207) 2024-11-13T18:36:59,907 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/6eea27e7601f46f9af4690b283cd8d22 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6eea27e7601f46f9af4690b283cd8d22 2024-11-13T18:36:59,918 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/C of 4f01271d10fc9cb8342007ba2bfde68b into 6eea27e7601f46f9af4690b283cd8d22(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:36:59,918 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:36:59,918 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/C, priority=13, startTime=1731523019027; duration=0sec 2024-11-13T18:36:59,918 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:36:59,918 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:C 2024-11-13T18:36:59,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741962_1138 (size=30955) 2024-11-13T18:37:00,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:37:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/3463b0d024fd4260a61abcd5b3299a24 2024-11-13T18:37:00,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/5890bc6974d64c63b13ac2be63230df1 is 50, key is test_row_0/B:col10/1731523018893/Put/seqid=0 2024-11-13T18:37:00,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523080064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523080064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523080074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523080074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523080084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:37:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741963_1139 (size=12001) 2024-11-13T18:37:00,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/5890bc6974d64c63b13ac2be63230df1 2024-11-13T18:37:00,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/a1f530ee16b342f9a07f5e15137aa532 is 50, key is test_row_0/C:col10/1731523018893/Put/seqid=0 2024-11-13T18:37:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:37:00,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741964_1140 (size=12001) 2024-11-13T18:37:00,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:37:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:37:00,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523080624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523080626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523080633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523080634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523080634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:00,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:37:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:00,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:00,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/a1f530ee16b342f9a07f5e15137aa532 2024-11-13T18:37:00,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/3463b0d024fd4260a61abcd5b3299a24 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/3463b0d024fd4260a61abcd5b3299a24 2024-11-13T18:37:00,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/3463b0d024fd4260a61abcd5b3299a24, entries=150, sequenceid=118, filesize=30.2 K 2024-11-13T18:37:00,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/5890bc6974d64c63b13ac2be63230df1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/5890bc6974d64c63b13ac2be63230df1 2024-11-13T18:37:00,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/5890bc6974d64c63b13ac2be63230df1, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T18:37:00,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/a1f530ee16b342f9a07f5e15137aa532 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a1f530ee16b342f9a07f5e15137aa532 2024-11-13T18:37:00,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a1f530ee16b342f9a07f5e15137aa532, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T18:37:00,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4f01271d10fc9cb8342007ba2bfde68b in 1612ms, sequenceid=118, compaction requested=false 2024-11-13T18:37:00,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:00,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T18:37:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:00,814 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T18:37:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111359d006bbc3d3490997b77f88daff461c_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523019235/Put/seqid=0 2024-11-13T18:37:00,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741965_1141 (size=12154) 2024-11-13T18:37:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:00,910 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111359d006bbc3d3490997b77f88daff461c_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359d006bbc3d3490997b77f88daff461c_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:00,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/60676741d41a4f6f9716c4c65cea6ca3, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:00,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/60676741d41a4f6f9716c4c65cea6ca3 is 175, key is test_row_0/A:col10/1731523019235/Put/seqid=0 2024-11-13T18:37:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741966_1142 (size=30955) 2024-11-13T18:37:00,957 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/60676741d41a4f6f9716c4c65cea6ca3 2024-11-13T18:37:00,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/fd4e69d1206348b09c9fff0e8fb59459 is 50, key is test_row_0/B:col10/1731523019235/Put/seqid=0 2024-11-13T18:37:01,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741967_1143 (size=12001) 2024-11-13T18:37:01,034 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/fd4e69d1206348b09c9fff0e8fb59459 2024-11-13T18:37:01,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/8993a98644fa43548568672af03dd0e1 is 50, key is test_row_0/C:col10/1731523019235/Put/seqid=0 2024-11-13T18:37:01,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741968_1144 (size=12001) 2024-11-13T18:37:01,110 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/8993a98644fa43548568672af03dd0e1 2024-11-13T18:37:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/60676741d41a4f6f9716c4c65cea6ca3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60676741d41a4f6f9716c4c65cea6ca3 2024-11-13T18:37:01,128 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60676741d41a4f6f9716c4c65cea6ca3, entries=150, sequenceid=131, filesize=30.2 K 2024-11-13T18:37:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/fd4e69d1206348b09c9fff0e8fb59459 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fd4e69d1206348b09c9fff0e8fb59459 2024-11-13T18:37:01,135 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fd4e69d1206348b09c9fff0e8fb59459, entries=150, sequenceid=131, filesize=11.7 K 2024-11-13T18:37:01,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/8993a98644fa43548568672af03dd0e1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8993a98644fa43548568672af03dd0e1 2024-11-13T18:37:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,143 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8993a98644fa43548568672af03dd0e1, entries=150, sequenceid=131, filesize=11.7 K 2024-11-13T18:37:01,144 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 4f01271d10fc9cb8342007ba2bfde68b in 330ms, sequenceid=131, compaction requested=true 2024-11-13T18:37:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-13T18:37:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-13T18:37:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T18:37:01,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9930 sec 2024-11-13T18:37:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.0000 sec 2024-11-13T18:37:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T18:37:01,304 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,305 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-13T18:37:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:01,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:37:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:37:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T18:37:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:01,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:01,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:01,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-13T18:37:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-13T18:37:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-13T18:37:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-13T18:37:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 175 msec 2024-11-13T18:37:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T18:37:01,645 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,646 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-13T18:37:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:37:01,650 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,651 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:01,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:01,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:37:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:37:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e9a6a2fa672647e79e46f18dedaa0e7b_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523021728/Put/seqid=0 2024-11-13T18:37:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741969_1145 (size=27248) 2024-11-13T18:37:01,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:37:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:01,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:01,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523081798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,808 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:01,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523081801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523081803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,816 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e9a6a2fa672647e79e46f18dedaa0e7b_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e9a6a2fa672647e79e46f18dedaa0e7b_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:01,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/0b686fec64484904b59d882205964981, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:01,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/0b686fec64484904b59d882205964981 is 175, key is test_row_0/A:col10/1731523021728/Put/seqid=0 2024-11-13T18:37:01,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523081807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741970_1146 (size=83035) 2024-11-13T18:37:01,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523081819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523081913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523081924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523081925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523081925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523081934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:01,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:37:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:01,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:37:02,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:37:02,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:02,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523082134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523082134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523082134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523082136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523082147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,234 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/0b686fec64484904b59d882205964981 2024-11-13T18:37:02,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/2abcfb9b5eeb40e281cda5b2890d87d0 is 50, key is test_row_0/B:col10/1731523021728/Put/seqid=0 2024-11-13T18:37:02,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:37:02,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:02,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:37:02,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741971_1147 (size=12151) 2024-11-13T18:37:02,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:37:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523082443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523082444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523082455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523082455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523082455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:37:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/2abcfb9b5eeb40e281cda5b2890d87d0 2024-11-13T18:37:02,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/1066b2e746b945ac86abcd9219434a79 is 50, key is test_row_0/C:col10/1731523021728/Put/seqid=0 2024-11-13T18:37:02,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:37:02,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:02,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:02,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741972_1148 (size=12151) 2024-11-13T18:37:02,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/1066b2e746b945ac86abcd9219434a79 2024-11-13T18:37:02,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/0b686fec64484904b59d882205964981 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0b686fec64484904b59d882205964981 2024-11-13T18:37:02,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0b686fec64484904b59d882205964981, entries=450, sequenceid=144, filesize=81.1 K 2024-11-13T18:37:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:37:02,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/2abcfb9b5eeb40e281cda5b2890d87d0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/2abcfb9b5eeb40e281cda5b2890d87d0 2024-11-13T18:37:02,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/2abcfb9b5eeb40e281cda5b2890d87d0, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T18:37:02,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/1066b2e746b945ac86abcd9219434a79 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1066b2e746b945ac86abcd9219434a79 2024-11-13T18:37:02,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1066b2e746b945ac86abcd9219434a79, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T18:37:02,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 4f01271d10fc9cb8342007ba2bfde68b in 1070ms, sequenceid=144, compaction requested=true 2024-11-13T18:37:02,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:02,802 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:02,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:02,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:02,803 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:02,804 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176213 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:02,804 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/A is initiating minor compaction (all files) 2024-11-13T18:37:02,804 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/A in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,804 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/11bbb39504f84e82ad07090676ef732e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/3463b0d024fd4260a61abcd5b3299a24, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60676741d41a4f6f9716c4c65cea6ca3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0b686fec64484904b59d882205964981] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=172.1 K 2024-11-13T18:37:02,804 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,804 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/11bbb39504f84e82ad07090676ef732e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/3463b0d024fd4260a61abcd5b3299a24, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60676741d41a4f6f9716c4c65cea6ca3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0b686fec64484904b59d882205964981] 2024-11-13T18:37:02,805 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11bbb39504f84e82ad07090676ef732e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523014603 2024-11-13T18:37:02,805 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:02,805 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/B is initiating minor compaction (all files) 2024-11-13T18:37:02,805 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/B in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,805 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fc41bd5fc72f4e55ae5e2db40c78ede0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/5890bc6974d64c63b13ac2be63230df1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fd4e69d1206348b09c9fff0e8fb59459, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/2abcfb9b5eeb40e281cda5b2890d87d0] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=47.2 K 2024-11-13T18:37:02,806 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3463b0d024fd4260a61abcd5b3299a24, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731523018850 2024-11-13T18:37:02,806 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting fc41bd5fc72f4e55ae5e2db40c78ede0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523014603 2024-11-13T18:37:02,806 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60676741d41a4f6f9716c4c65cea6ca3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731523019150 2024-11-13T18:37:02,807 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5890bc6974d64c63b13ac2be63230df1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731523018850 2024-11-13T18:37:02,808 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b686fec64484904b59d882205964981, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731523021691 2024-11-13T18:37:02,809 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting fd4e69d1206348b09c9fff0e8fb59459, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731523019150 2024-11-13T18:37:02,810 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2abcfb9b5eeb40e281cda5b2890d87d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731523021698 2024-11-13T18:37:02,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:02,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:02,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:02,824 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:02,837 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#B#compaction#124 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:02,838 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/6d6d5bbed7d24e06acbb8aedba536ab8 is 50, key is test_row_0/B:col10/1731523021728/Put/seqid=0 2024-11-13T18:37:02,843 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111305ccc9e5d9744dbfaaca39c8dd7c3b52_4f01271d10fc9cb8342007ba2bfde68b store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:02,848 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111305ccc9e5d9744dbfaaca39c8dd7c3b52_4f01271d10fc9cb8342007ba2bfde68b, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:02,848 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111305ccc9e5d9744dbfaaca39c8dd7c3b52_4f01271d10fc9cb8342007ba2bfde68b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:02,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741973_1149 (size=12493) 2024-11-13T18:37:02,878 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/6d6d5bbed7d24e06acbb8aedba536ab8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/6d6d5bbed7d24e06acbb8aedba536ab8 2024-11-13T18:37:02,885 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/B of 4f01271d10fc9cb8342007ba2bfde68b into 6d6d5bbed7d24e06acbb8aedba536ab8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:02,885 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:02,885 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/B, priority=12, startTime=1731523022802; duration=0sec 2024-11-13T18:37:02,886 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:02,886 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:B 2024-11-13T18:37:02,886 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:02,888 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:02,888 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/C is initiating minor compaction (all files) 2024-11-13T18:37:02,888 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/C in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,888 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6eea27e7601f46f9af4690b283cd8d22, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a1f530ee16b342f9a07f5e15137aa532, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8993a98644fa43548568672af03dd0e1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1066b2e746b945ac86abcd9219434a79] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=47.2 K 2024-11-13T18:37:02,889 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eea27e7601f46f9af4690b283cd8d22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523014603 2024-11-13T18:37:02,889 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f530ee16b342f9a07f5e15137aa532, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731523018850 2024-11-13T18:37:02,890 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8993a98644fa43548568672af03dd0e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731523019150 2024-11-13T18:37:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741974_1150 (size=4469) 2024-11-13T18:37:02,891 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1066b2e746b945ac86abcd9219434a79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731523021698 2024-11-13T18:37:02,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T18:37:02,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:02,892 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:37:02,904 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#A#compaction#123 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:02,905 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/185236dcad9f473c83ac7b8e0d634ad2 is 175, key is test_row_0/A:col10/1731523021728/Put/seqid=0 2024-11-13T18:37:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371caa8339583413785c42db3657d92e4_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523021800/Put/seqid=0 2024-11-13T18:37:02,925 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#C#compaction#126 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:02,925 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/a38da16c22bf48d5b1055d6e341695d5 is 50, key is test_row_0/C:col10/1731523021728/Put/seqid=0 2024-11-13T18:37:02,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741975_1151 (size=31447) 2024-11-13T18:37:02,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741977_1153 (size=12493) 2024-11-13T18:37:02,946 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/185236dcad9f473c83ac7b8e0d634ad2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/185236dcad9f473c83ac7b8e0d634ad2 2024-11-13T18:37:02,946 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/a38da16c22bf48d5b1055d6e341695d5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a38da16c22bf48d5b1055d6e341695d5 2024-11-13T18:37:02,954 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/C of 4f01271d10fc9cb8342007ba2bfde68b into a38da16c22bf48d5b1055d6e341695d5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:02,954 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:02,954 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/C, priority=12, startTime=1731523022810; duration=0sec 2024-11-13T18:37:02,954 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:02,954 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:C 2024-11-13T18:37:02,956 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/A of 4f01271d10fc9cb8342007ba2bfde68b into 185236dcad9f473c83ac7b8e0d634ad2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:02,956 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:02,956 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/A, priority=12, startTime=1731523022801; duration=0sec 2024-11-13T18:37:02,956 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:02,956 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:A 2024-11-13T18:37:02,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741976_1152 (size=12304) 2024-11-13T18:37:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:02,967 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371caa8339583413785c42db3657d92e4_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371caa8339583413785c42db3657d92e4_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:02,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:02,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/9808d6314b85404b8b8c9e23909cae84, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/9808d6314b85404b8b8c9e23909cae84 is 175, key is test_row_0/A:col10/1731523021800/Put/seqid=0 2024-11-13T18:37:02,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523082975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523082975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523082981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523082982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523082983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:02,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741978_1154 (size=31105) 2024-11-13T18:37:02,999 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/9808d6314b85404b8b8c9e23909cae84 2024-11-13T18:37:03,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/1501a3b7ce0a4f659af4188afe05962b is 50, key is test_row_0/B:col10/1731523021800/Put/seqid=0 2024-11-13T18:37:03,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741979_1155 (size=12151) 2024-11-13T18:37:03,025 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/1501a3b7ce0a4f659af4188afe05962b 2024-11-13T18:37:03,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/2eaf6ba69442462da752f81fea162e6d is 50, key is test_row_0/C:col10/1731523021800/Put/seqid=0 2024-11-13T18:37:03,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741980_1156 (size=12151) 2024-11-13T18:37:03,055 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/2eaf6ba69442462da752f81fea162e6d 2024-11-13T18:37:03,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/9808d6314b85404b8b8c9e23909cae84 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9808d6314b85404b8b8c9e23909cae84 2024-11-13T18:37:03,070 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9808d6314b85404b8b8c9e23909cae84, entries=150, sequenceid=170, filesize=30.4 K 2024-11-13T18:37:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/1501a3b7ce0a4f659af4188afe05962b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1501a3b7ce0a4f659af4188afe05962b 2024-11-13T18:37:03,082 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1501a3b7ce0a4f659af4188afe05962b, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T18:37:03,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/2eaf6ba69442462da752f81fea162e6d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2eaf6ba69442462da752f81fea162e6d 2024-11-13T18:37:03,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523083094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523083094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,095 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2eaf6ba69442462da752f81fea162e6d, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T18:37:03,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,096 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4f01271d10fc9cb8342007ba2bfde68b in 204ms, sequenceid=170, compaction requested=false 2024-11-13T18:37:03,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:03,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:03,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T18:37:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T18:37:03,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T18:37:03,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-13T18:37:03,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.4550 sec 2024-11-13T18:37:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:03,107 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:03,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:37:03,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133d4539fbe18749a686932de459737e96_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523023105/Put/seqid=0 2024-11-13T18:37:03,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741981_1157 (size=12304) 2024-11-13T18:37:03,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523083175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523083175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523083181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523083284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523083293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523083294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523083304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523083306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523083504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523083504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523083504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,555 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:03,563 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133d4539fbe18749a686932de459737e96_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133d4539fbe18749a686932de459737e96_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:03,564 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/45207759ae8d4be2b1abdd1f81bbee96, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:03,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/45207759ae8d4be2b1abdd1f81bbee96 is 175, key is test_row_0/A:col10/1731523023105/Put/seqid=0 2024-11-13T18:37:03,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741982_1158 (size=31105) 2024-11-13T18:37:03,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/45207759ae8d4be2b1abdd1f81bbee96 2024-11-13T18:37:03,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/df19b158b78a444f92cdd8f9b0dfaedb is 50, key is test_row_0/B:col10/1731523023105/Put/seqid=0 2024-11-13T18:37:03,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523083613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523083613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741983_1159 (size=12151) 2024-11-13T18:37:03,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/df19b158b78a444f92cdd8f9b0dfaedb 2024-11-13T18:37:03,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/bdae0cdca7c248a4af98f87bdd4e536c is 50, key is test_row_0/C:col10/1731523023105/Put/seqid=0 2024-11-13T18:37:03,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741984_1160 (size=12151) 2024-11-13T18:37:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T18:37:03,811 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:03,812 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T18:37:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523083815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:37:03,824 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:03,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523083815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523083815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:03,829 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:03,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:37:03,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T18:37:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:03,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:04,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/bdae0cdca7c248a4af98f87bdd4e536c 2024-11-13T18:37:04,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/45207759ae8d4be2b1abdd1f81bbee96 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/45207759ae8d4be2b1abdd1f81bbee96 2024-11-13T18:37:04,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/45207759ae8d4be2b1abdd1f81bbee96, entries=150, sequenceid=184, filesize=30.4 K 2024-11-13T18:37:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/df19b158b78a444f92cdd8f9b0dfaedb as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/df19b158b78a444f92cdd8f9b0dfaedb 2024-11-13T18:37:04,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/df19b158b78a444f92cdd8f9b0dfaedb, entries=150, sequenceid=184, filesize=11.9 K 2024-11-13T18:37:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523084123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/bdae0cdca7c248a4af98f87bdd4e536c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/bdae0cdca7c248a4af98f87bdd4e536c 2024-11-13T18:37:04,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T18:37:04,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:04,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:04,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523084136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:04,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:37:04,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/bdae0cdca7c248a4af98f87bdd4e536c, entries=150, sequenceid=184, filesize=11.9 K 2024-11-13T18:37:04,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4f01271d10fc9cb8342007ba2bfde68b in 1058ms, sequenceid=184, compaction requested=true 2024-11-13T18:37:04,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:04,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:04,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:04,166 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:04,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:04,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:04,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:04,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:37:04,169 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:04,177 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:04,177 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/A is initiating minor compaction (all files) 2024-11-13T18:37:04,177 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/A in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:04,177 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/185236dcad9f473c83ac7b8e0d634ad2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9808d6314b85404b8b8c9e23909cae84, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/45207759ae8d4be2b1abdd1f81bbee96] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=91.5 K 2024-11-13T18:37:04,177 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:04,177 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/185236dcad9f473c83ac7b8e0d634ad2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9808d6314b85404b8b8c9e23909cae84, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/45207759ae8d4be2b1abdd1f81bbee96] 2024-11-13T18:37:04,181 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185236dcad9f473c83ac7b8e0d634ad2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731523021698 2024-11-13T18:37:04,182 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:04,182 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/B is initiating minor compaction (all files) 2024-11-13T18:37:04,182 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/B in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:04,182 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/6d6d5bbed7d24e06acbb8aedba536ab8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1501a3b7ce0a4f659af4188afe05962b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/df19b158b78a444f92cdd8f9b0dfaedb] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=35.9 K 2024-11-13T18:37:04,182 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9808d6314b85404b8b8c9e23909cae84, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523021788 2024-11-13T18:37:04,185 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d6d5bbed7d24e06acbb8aedba536ab8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731523021698 2024-11-13T18:37:04,186 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45207759ae8d4be2b1abdd1f81bbee96, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731523022972 2024-11-13T18:37:04,189 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1501a3b7ce0a4f659af4188afe05962b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523021788 2024-11-13T18:37:04,190 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting df19b158b78a444f92cdd8f9b0dfaedb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731523022972 2024-11-13T18:37:04,227 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:04,241 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138caedeaa7fbb41abb3f16fb389276ea9_4f01271d10fc9cb8342007ba2bfde68b store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:04,243 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138caedeaa7fbb41abb3f16fb389276ea9_4f01271d10fc9cb8342007ba2bfde68b, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:04,243 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138caedeaa7fbb41abb3f16fb389276ea9_4f01271d10fc9cb8342007ba2bfde68b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:04,276 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#B#compaction#133 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:04,276 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/3e63fbeef50a4f1aa1b9f206b5c128b3 is 50, key is test_row_0/B:col10/1731523023105/Put/seqid=0 2024-11-13T18:37:04,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T18:37:04,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:04,294 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T18:37:04,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741985_1161 (size=4469) 2024-11-13T18:37:04,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741986_1162 (size=12595) 2024-11-13T18:37:04,336 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/3e63fbeef50a4f1aa1b9f206b5c128b3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3e63fbeef50a4f1aa1b9f206b5c128b3 2024-11-13T18:37:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:04,349 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/B of 4f01271d10fc9cb8342007ba2bfde68b into 3e63fbeef50a4f1aa1b9f206b5c128b3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:04,349 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:04,349 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/B, priority=13, startTime=1731523024166; duration=0sec 2024-11-13T18:37:04,349 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:04,349 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:B 2024-11-13T18:37:04,350 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:04,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:04,351 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:04,351 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/C is initiating minor compaction (all files) 2024-11-13T18:37:04,351 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/C in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:04,351 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a38da16c22bf48d5b1055d6e341695d5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2eaf6ba69442462da752f81fea162e6d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/bdae0cdca7c248a4af98f87bdd4e536c] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=35.9 K 2024-11-13T18:37:04,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c1fda2d96a74725a103f0e448764a81_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523023165/Put/seqid=0 2024-11-13T18:37:04,358 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a38da16c22bf48d5b1055d6e341695d5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731523021698 2024-11-13T18:37:04,362 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eaf6ba69442462da752f81fea162e6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523021788 2024-11-13T18:37:04,364 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting bdae0cdca7c248a4af98f87bdd4e536c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731523022972 2024-11-13T18:37:04,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523084376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523084378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523084380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741987_1163 (size=12304) 2024-11-13T18:37:04,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:04,427 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:04,428 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/8a24d89cbf4d40ba940f9688ec3d9f10 is 50, key is test_row_0/C:col10/1731523023105/Put/seqid=0 2024-11-13T18:37:04,455 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c1fda2d96a74725a103f0e448764a81_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c1fda2d96a74725a103f0e448764a81_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:37:04,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/a7d591727b4f4c14843cf4e56459c69f, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:04,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741988_1164 (size=12595) 2024-11-13T18:37:04,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/a7d591727b4f4c14843cf4e56459c69f is 175, key is test_row_0/A:col10/1731523023165/Put/seqid=0 2024-11-13T18:37:04,503 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/8a24d89cbf4d40ba940f9688ec3d9f10 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8a24d89cbf4d40ba940f9688ec3d9f10 2024-11-13T18:37:04,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523084498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523084498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741989_1165 (size=31105) 2024-11-13T18:37:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523084506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,524 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/C of 4f01271d10fc9cb8342007ba2bfde68b into 8a24d89cbf4d40ba940f9688ec3d9f10(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:04,524 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:04,524 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/C, priority=13, startTime=1731523024166; duration=0sec 2024-11-13T18:37:04,524 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:04,524 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:C 2024-11-13T18:37:04,705 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#A#compaction#132 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:04,706 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/cf7ae74c4f08484a9646527a0a04debc is 175, key is test_row_0/A:col10/1731523023105/Put/seqid=0 2024-11-13T18:37:04,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523084718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523084719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741990_1166 (size=31549) 2024-11-13T18:37:04,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523084732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:04,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:04,914 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/a7d591727b4f4c14843cf4e56459c69f 2024-11-13T18:37:04,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/97992b7bab414d9fa711e4b7d73b1913 is 50, key is test_row_0/B:col10/1731523023165/Put/seqid=0 2024-11-13T18:37:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:37:04,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741991_1167 (size=12151) 2024-11-13T18:37:05,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523085033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:05,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523085033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523085054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:05,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523085146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:05,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523085146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:05,151 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 4f01271d10fc9cb8342007ba2bfde68b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,151 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 4f01271d10fc9cb8342007ba2bfde68b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:05,164 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/cf7ae74c4f08484a9646527a0a04debc as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/cf7ae74c4f08484a9646527a0a04debc 2024-11-13T18:37:05,171 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/A of 4f01271d10fc9cb8342007ba2bfde68b into cf7ae74c4f08484a9646527a0a04debc(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:05,171 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:05,171 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/A, priority=13, startTime=1731523024165; duration=0sec 2024-11-13T18:37:05,171 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:05,171 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:A 2024-11-13T18:37:05,394 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/97992b7bab414d9fa711e4b7d73b1913 2024-11-13T18:37:05,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/529a802a4c5542a7b8903c7f7cfcc4a5 is 50, key is test_row_0/C:col10/1731523023165/Put/seqid=0 2024-11-13T18:37:05,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741992_1168 (size=12151) 2024-11-13T18:37:05,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523085543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:05,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523085558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:05,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523085573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:05,870 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/529a802a4c5542a7b8903c7f7cfcc4a5 2024-11-13T18:37:05,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/a7d591727b4f4c14843cf4e56459c69f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/a7d591727b4f4c14843cf4e56459c69f 2024-11-13T18:37:05,882 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/a7d591727b4f4c14843cf4e56459c69f, entries=150, sequenceid=208, filesize=30.4 K 2024-11-13T18:37:05,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/97992b7bab414d9fa711e4b7d73b1913 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/97992b7bab414d9fa711e4b7d73b1913 2024-11-13T18:37:05,913 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/97992b7bab414d9fa711e4b7d73b1913, entries=150, sequenceid=208, filesize=11.9 K 2024-11-13T18:37:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/529a802a4c5542a7b8903c7f7cfcc4a5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/529a802a4c5542a7b8903c7f7cfcc4a5 2024-11-13T18:37:05,920 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/529a802a4c5542a7b8903c7f7cfcc4a5, entries=150, sequenceid=208, filesize=11.9 K 2024-11-13T18:37:05,922 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4f01271d10fc9cb8342007ba2bfde68b in 1628ms, sequenceid=208, compaction requested=false 2024-11-13T18:37:05,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:05,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:05,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T18:37:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T18:37:05,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T18:37:05,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1020 sec 2024-11-13T18:37:05,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.1300 sec 2024-11-13T18:37:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T18:37:06,004 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:06,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T18:37:06,007 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:37:06,009 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:06,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:37:06,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T18:37:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:06,170 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:37:06,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134d1bcac0eba048ec829b4c919d492320_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523024378/Put/seqid=0 2024-11-13T18:37:06,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741993_1169 (size=12304) 2024-11-13T18:37:06,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:06,265 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134d1bcac0eba048ec829b4c919d492320_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134d1bcac0eba048ec829b4c919d492320_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:06,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/c232e94063e847bb8065a925e2d7fef0, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:06,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/c232e94063e847bb8065a925e2d7fef0 is 175, key is test_row_0/A:col10/1731523024378/Put/seqid=0 2024-11-13T18:37:06,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741994_1170 (size=31105) 2024-11-13T18:37:06,307 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/c232e94063e847bb8065a925e2d7fef0 2024-11-13T18:37:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:37:06,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/e65ad04750b9440589badffbd77a2ee0 is 50, key is test_row_0/B:col10/1731523024378/Put/seqid=0 2024-11-13T18:37:06,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741995_1171 (size=12151) 2024-11-13T18:37:06,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:37:06,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:06,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523086669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523086670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:06,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523086683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:06,792 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/e65ad04750b9440589badffbd77a2ee0 2024-11-13T18:37:06,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523086794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:06,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523086794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:06,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523086813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:06,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/0a34b2e6f6f24adc857622e984c4b49f is 50, key is test_row_0/C:col10/1731523024378/Put/seqid=0 2024-11-13T18:37:06,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741996_1172 (size=12151) 2024-11-13T18:37:07,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523087009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523087018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523087029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:37:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523087166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523087177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,180 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 4f01271d10fc9cb8342007ba2bfde68b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,181 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 4f01271d10fc9cb8342007ba2bfde68b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,326 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/0a34b2e6f6f24adc857622e984c4b49f 2024-11-13T18:37:07,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523087324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523087333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/c232e94063e847bb8065a925e2d7fef0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/c232e94063e847bb8065a925e2d7fef0 2024-11-13T18:37:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523087358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,371 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/c232e94063e847bb8065a925e2d7fef0, entries=150, sequenceid=223, filesize=30.4 K 2024-11-13T18:37:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/e65ad04750b9440589badffbd77a2ee0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/e65ad04750b9440589badffbd77a2ee0 2024-11-13T18:37:07,378 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/e65ad04750b9440589badffbd77a2ee0, entries=150, sequenceid=223, filesize=11.9 K 2024-11-13T18:37:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/0a34b2e6f6f24adc857622e984c4b49f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/0a34b2e6f6f24adc857622e984c4b49f 2024-11-13T18:37:07,386 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/0a34b2e6f6f24adc857622e984c4b49f, entries=150, sequenceid=223, filesize=11.9 K 2024-11-13T18:37:07,388 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4f01271d10fc9cb8342007ba2bfde68b in 1218ms, sequenceid=223, compaction requested=true 2024-11-13T18:37:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T18:37:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T18:37:07,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T18:37:07,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-11-13T18:37:07,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4240 sec 2024-11-13T18:37:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:07,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:37:07,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111339e6aa6fb0c94969aef6cadaa2c8c0e7_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523026658/Put/seqid=0 2024-11-13T18:37:07,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523087902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523087906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523087914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:07,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741997_1173 (size=12304) 2024-11-13T18:37:07,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:07,973 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111339e6aa6fb0c94969aef6cadaa2c8c0e7_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111339e6aa6fb0c94969aef6cadaa2c8c0e7_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:07,974 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/60a27beb00d54157bf8585734d2e74c7, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:07,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/60a27beb00d54157bf8585734d2e74c7 is 175, key is test_row_0/A:col10/1731523026658/Put/seqid=0 2024-11-13T18:37:08,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523088013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523088014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741998_1174 (size=31105) 2024-11-13T18:37:08,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523088033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T18:37:08,163 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:08,164 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-13T18:37:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:37:08,176 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:08,179 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:08,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:08,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523088225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523088225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523088245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:37:08,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:37:08,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:08,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,427 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/60a27beb00d54157bf8585734d2e74c7 2024-11-13T18:37:08,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/9fdb77fdae5648f5bff7b285dcdf458d is 50, key is test_row_0/B:col10/1731523026658/Put/seqid=0 2024-11-13T18:37:08,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741999_1175 (size=12151) 2024-11-13T18:37:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:37:08,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:37:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523088533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523088533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523088564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:08,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:37:08,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:08,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:37:08,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:37:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/9fdb77fdae5648f5bff7b285dcdf458d 2024-11-13T18:37:08,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/457c880ebfeb4c628a61cd3ddadbac0e is 50, key is test_row_0/C:col10/1731523026658/Put/seqid=0 2024-11-13T18:37:08,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742000_1176 (size=12151) 2024-11-13T18:37:08,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:37:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:08,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:09,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523089044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:09,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523089053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:09,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523089076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:09,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:37:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:09,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:37:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:09,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:37:09,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/457c880ebfeb4c628a61cd3ddadbac0e 2024-11-13T18:37:09,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/60a27beb00d54157bf8585734d2e74c7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60a27beb00d54157bf8585734d2e74c7 2024-11-13T18:37:09,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60a27beb00d54157bf8585734d2e74c7, entries=150, sequenceid=248, filesize=30.4 K 2024-11-13T18:37:09,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/9fdb77fdae5648f5bff7b285dcdf458d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/9fdb77fdae5648f5bff7b285dcdf458d 2024-11-13T18:37:09,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/9fdb77fdae5648f5bff7b285dcdf458d, entries=150, sequenceid=248, filesize=11.9 K 2024-11-13T18:37:09,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/457c880ebfeb4c628a61cd3ddadbac0e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/457c880ebfeb4c628a61cd3ddadbac0e 2024-11-13T18:37:09,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/457c880ebfeb4c628a61cd3ddadbac0e, entries=150, sequenceid=248, filesize=11.9 K 2024-11-13T18:37:09,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4f01271d10fc9cb8342007ba2bfde68b in 1549ms, sequenceid=248, compaction requested=true 2024-11-13T18:37:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:09,406 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:09,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:09,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:09,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:37:09,408 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:09,408 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124864 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:09,408 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/A is initiating minor compaction (all files) 2024-11-13T18:37:09,408 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/A in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,408 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/cf7ae74c4f08484a9646527a0a04debc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/a7d591727b4f4c14843cf4e56459c69f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/c232e94063e847bb8065a925e2d7fef0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60a27beb00d54157bf8585734d2e74c7] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=121.9 K 2024-11-13T18:37:09,408 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,408 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/cf7ae74c4f08484a9646527a0a04debc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/a7d591727b4f4c14843cf4e56459c69f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/c232e94063e847bb8065a925e2d7fef0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60a27beb00d54157bf8585734d2e74c7] 2024-11-13T18:37:09,409 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7ae74c4f08484a9646527a0a04debc, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731523022972 2024-11-13T18:37:09,410 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d591727b4f4c14843cf4e56459c69f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731523023165 2024-11-13T18:37:09,410 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c232e94063e847bb8065a925e2d7fef0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731523024360 2024-11-13T18:37:09,410 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:09,410 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/B is initiating minor compaction (all files) 2024-11-13T18:37:09,410 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/B in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,411 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3e63fbeef50a4f1aa1b9f206b5c128b3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/97992b7bab414d9fa711e4b7d73b1913, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/e65ad04750b9440589badffbd77a2ee0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/9fdb77fdae5648f5bff7b285dcdf458d] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=47.9 K 2024-11-13T18:37:09,411 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e63fbeef50a4f1aa1b9f206b5c128b3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731523022972 2024-11-13T18:37:09,412 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 97992b7bab414d9fa711e4b7d73b1913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731523023165 2024-11-13T18:37:09,413 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e65ad04750b9440589badffbd77a2ee0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731523024360 2024-11-13T18:37:09,413 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60a27beb00d54157bf8585734d2e74c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731523026658 2024-11-13T18:37:09,413 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fdb77fdae5648f5bff7b285dcdf458d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731523026658 2024-11-13T18:37:09,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T18:37:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,434 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:09,454 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:09,468 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:09,469 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/62964ca9770240aa83c9757a81435e6c is 50, key is test_row_0/B:col10/1731523026658/Put/seqid=0 2024-11-13T18:37:09,478 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411133a1e70330e4643949130221a0fc1dbff_4f01271d10fc9cb8342007ba2bfde68b store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:09,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111302e1efc373264a78b96e73f524222cca_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523027886/Put/seqid=0 2024-11-13T18:37:09,490 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411133a1e70330e4643949130221a0fc1dbff_4f01271d10fc9cb8342007ba2bfde68b, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:09,490 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133a1e70330e4643949130221a0fc1dbff_4f01271d10fc9cb8342007ba2bfde68b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:09,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742001_1177 (size=12731) 2024-11-13T18:37:09,517 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/62964ca9770240aa83c9757a81435e6c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/62964ca9770240aa83c9757a81435e6c 2024-11-13T18:37:09,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742003_1179 (size=4469) 2024-11-13T18:37:09,523 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#A#compaction#144 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:09,524 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/2570c955725c4274bafc8289e3e3efc0 is 175, key is test_row_0/A:col10/1731523026658/Put/seqid=0 2024-11-13T18:37:09,527 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/B of 4f01271d10fc9cb8342007ba2bfde68b into 62964ca9770240aa83c9757a81435e6c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:09,527 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:09,527 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/B, priority=12, startTime=1731523029406; duration=0sec 2024-11-13T18:37:09,527 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:09,527 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:B 2024-11-13T18:37:09,527 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:09,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742002_1178 (size=12404) 2024-11-13T18:37:09,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:09,530 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:09,530 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/C is initiating minor compaction (all files) 2024-11-13T18:37:09,530 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/C in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:09,531 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8a24d89cbf4d40ba940f9688ec3d9f10, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/529a802a4c5542a7b8903c7f7cfcc4a5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/0a34b2e6f6f24adc857622e984c4b49f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/457c880ebfeb4c628a61cd3ddadbac0e] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=47.9 K 2024-11-13T18:37:09,532 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a24d89cbf4d40ba940f9688ec3d9f10, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731523022972 2024-11-13T18:37:09,533 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 529a802a4c5542a7b8903c7f7cfcc4a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731523023165 2024-11-13T18:37:09,533 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a34b2e6f6f24adc857622e984c4b49f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731523024360 2024-11-13T18:37:09,534 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 457c880ebfeb4c628a61cd3ddadbac0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731523026658 2024-11-13T18:37:09,534 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111302e1efc373264a78b96e73f524222cca_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302e1efc373264a78b96e73f524222cca_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:09,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/0907a75e83cd47ac8ae0e5b14cb288f1, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:09,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/0907a75e83cd47ac8ae0e5b14cb288f1 is 175, key is test_row_0/A:col10/1731523027886/Put/seqid=0 2024-11-13T18:37:09,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742004_1180 (size=31685) 2024-11-13T18:37:09,567 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/2570c955725c4274bafc8289e3e3efc0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/2570c955725c4274bafc8289e3e3efc0 2024-11-13T18:37:09,575 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:09,575 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/d9640928c6db4945a325601ea360af2b is 50, key is test_row_0/C:col10/1731523026658/Put/seqid=0 2024-11-13T18:37:09,579 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/A of 4f01271d10fc9cb8342007ba2bfde68b into 2570c955725c4274bafc8289e3e3efc0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:09,579 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:09,580 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/A, priority=12, startTime=1731523029406; duration=0sec 2024-11-13T18:37:09,580 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:09,580 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:A 2024-11-13T18:37:09,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742005_1181 (size=31205) 2024-11-13T18:37:09,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742006_1182 (size=12731) 2024-11-13T18:37:09,983 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/0907a75e83cd47ac8ae0e5b14cb288f1 2024-11-13T18:37:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/50538dc8aff148a6b5eeff0dc29b5ca7 is 50, key is test_row_0/B:col10/1731523027886/Put/seqid=0 2024-11-13T18:37:10,011 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/d9640928c6db4945a325601ea360af2b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/d9640928c6db4945a325601ea360af2b 2024-11-13T18:37:10,028 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/C of 4f01271d10fc9cb8342007ba2bfde68b into d9640928c6db4945a325601ea360af2b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:10,028 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:10,028 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/C, priority=12, startTime=1731523029406; duration=0sec 2024-11-13T18:37:10,028 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:10,028 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:C 2024-11-13T18:37:10,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742007_1183 (size=12251) 2024-11-13T18:37:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. as already flushing 2024-11-13T18:37:10,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523090150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523090151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523090159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523090273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523090273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523090274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:37:10,433 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/50538dc8aff148a6b5eeff0dc29b5ca7 2024-11-13T18:37:10,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/40bb6fe404514216aefb51c3110742cd is 50, key is test_row_0/C:col10/1731523027886/Put/seqid=0 2024-11-13T18:37:10,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742008_1184 (size=12251) 2024-11-13T18:37:10,455 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/40bb6fe404514216aefb51c3110742cd 2024-11-13T18:37:10,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/0907a75e83cd47ac8ae0e5b14cb288f1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0907a75e83cd47ac8ae0e5b14cb288f1 2024-11-13T18:37:10,474 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0907a75e83cd47ac8ae0e5b14cb288f1, entries=150, sequenceid=259, filesize=30.5 K 2024-11-13T18:37:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/50538dc8aff148a6b5eeff0dc29b5ca7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/50538dc8aff148a6b5eeff0dc29b5ca7 2024-11-13T18:37:10,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523090484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,491 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/50538dc8aff148a6b5eeff0dc29b5ca7, entries=150, sequenceid=259, filesize=12.0 K 2024-11-13T18:37:10,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/40bb6fe404514216aefb51c3110742cd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/40bb6fe404514216aefb51c3110742cd 2024-11-13T18:37:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523090484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523090484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,498 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/40bb6fe404514216aefb51c3110742cd, entries=150, sequenceid=259, filesize=12.0 K 2024-11-13T18:37:10,507 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4f01271d10fc9cb8342007ba2bfde68b in 1074ms, sequenceid=259, compaction requested=false 2024-11-13T18:37:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-13T18:37:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-13T18:37:10,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T18:37:10,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3420 sec 2024-11-13T18:37:10,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.3710 sec 2024-11-13T18:37:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:10,811 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:10,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T18:37:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523090846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523090867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135332555191c846d3bd91ed6033071303_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523030808/Put/seqid=0 2024-11-13T18:37:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523090875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,900 INFO [Thread-577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-577. 2024-11-13T18:37:10,900 DEBUG [Thread-577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:10,900 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:10,901 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:10,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:10,903 INFO [Thread-579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-579. 2024-11-13T18:37:10,903 DEBUG [Thread-579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:10,903 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:10,903 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:10,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:10,906 INFO [Thread-581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-581. 2024-11-13T18:37:10,906 DEBUG [Thread-581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:10,906 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:10,906 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:10,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:10,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742009_1185 (size=14994) 2024-11-13T18:37:10,927 INFO [Thread-583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-583. 2024-11-13T18:37:10,927 DEBUG [Thread-583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:10,927 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:10,933 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:10,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:10,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523090978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523090978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523090998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523091183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60486 deadline: 1731523091193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523091193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,194 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 4f01271d10fc9cb8342007ba2bfde68b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:11,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523091204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60534 deadline: 1731523091204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,204 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 4f01271d10fc9cb8342007ba2bfde68b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:11,328 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135332555191c846d3bd91ed6033071303_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135332555191c846d3bd91ed6033071303_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:11,329 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/f7890f8c784840e9b1aa59a47d87fd93, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:11,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/f7890f8c784840e9b1aa59a47d87fd93 is 175, key is test_row_0/A:col10/1731523030808/Put/seqid=0 2024-11-13T18:37:11,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742010_1186 (size=39949) 2024-11-13T18:37:11,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523091494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523091504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523091514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:11,736 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/f7890f8c784840e9b1aa59a47d87fd93 2024-11-13T18:37:11,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/f056d3dd69de4f25900d78f70d9b4a30 is 50, key is test_row_0/B:col10/1731523030808/Put/seqid=0 2024-11-13T18:37:11,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742011_1187 (size=12301) 2024-11-13T18:37:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60490 deadline: 1731523092004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:12,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60526 deadline: 1731523092013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:12,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60536 deadline: 1731523092023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:12,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/f056d3dd69de4f25900d78f70d9b4a30 2024-11-13T18:37:12,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/6952fe3b423b44e6a1caf5ec2a0f093f is 50, key is test_row_0/C:col10/1731523030808/Put/seqid=0 2024-11-13T18:37:12,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742012_1188 (size=12301) 2024-11-13T18:37:12,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/6952fe3b423b44e6a1caf5ec2a0f093f 2024-11-13T18:37:12,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/f7890f8c784840e9b1aa59a47d87fd93 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f7890f8c784840e9b1aa59a47d87fd93 2024-11-13T18:37:12,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f7890f8c784840e9b1aa59a47d87fd93, entries=200, sequenceid=290, filesize=39.0 K 2024-11-13T18:37:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/f056d3dd69de4f25900d78f70d9b4a30 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/f056d3dd69de4f25900d78f70d9b4a30 2024-11-13T18:37:12,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/f056d3dd69de4f25900d78f70d9b4a30, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T18:37:12,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/6952fe3b423b44e6a1caf5ec2a0f093f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6952fe3b423b44e6a1caf5ec2a0f093f 2024-11-13T18:37:12,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6952fe3b423b44e6a1caf5ec2a0f093f, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T18:37:12,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 4f01271d10fc9cb8342007ba2bfde68b in 1390ms, sequenceid=290, compaction requested=true 2024-11-13T18:37:12,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:12,201 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:12,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:12,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:12,202 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:12,202 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:12,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:12,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:12,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4f01271d10fc9cb8342007ba2bfde68b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:12,202 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/A is initiating minor compaction (all files) 2024-11-13T18:37:12,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:12,203 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/A in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:12,203 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/2570c955725c4274bafc8289e3e3efc0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0907a75e83cd47ac8ae0e5b14cb288f1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f7890f8c784840e9b1aa59a47d87fd93] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=100.4 K 2024-11-13T18:37:12,203 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:12,203 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/2570c955725c4274bafc8289e3e3efc0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0907a75e83cd47ac8ae0e5b14cb288f1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f7890f8c784840e9b1aa59a47d87fd93] 2024-11-13T18:37:12,203 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2570c955725c4274bafc8289e3e3efc0, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731523026658 2024-11-13T18:37:12,204 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0907a75e83cd47ac8ae0e5b14cb288f1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731523027886 2024-11-13T18:37:12,204 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:12,204 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/B is initiating minor compaction (all files) 2024-11-13T18:37:12,204 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/B in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:12,204 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/62964ca9770240aa83c9757a81435e6c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/50538dc8aff148a6b5eeff0dc29b5ca7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/f056d3dd69de4f25900d78f70d9b4a30] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=36.4 K 2024-11-13T18:37:12,204 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7890f8c784840e9b1aa59a47d87fd93, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731523030150 2024-11-13T18:37:12,205 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 62964ca9770240aa83c9757a81435e6c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731523026658 2024-11-13T18:37:12,205 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 50538dc8aff148a6b5eeff0dc29b5ca7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731523027886 2024-11-13T18:37:12,206 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting f056d3dd69de4f25900d78f70d9b4a30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731523030150 2024-11-13T18:37:12,218 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:12,222 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:12,223 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/1174de23e4c04248a59665348fee54ff is 50, key is test_row_0/B:col10/1731523030808/Put/seqid=0 2024-11-13T18:37:12,230 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113bde4f0f307994deca4f3121b3c5e4430_4f01271d10fc9cb8342007ba2bfde68b store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:12,236 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113bde4f0f307994deca4f3121b3c5e4430_4f01271d10fc9cb8342007ba2bfde68b, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:12,236 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bde4f0f307994deca4f3121b3c5e4430_4f01271d10fc9cb8342007ba2bfde68b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:12,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742013_1189 (size=12983) 2024-11-13T18:37:12,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742014_1190 (size=4469) 2024-11-13T18:37:12,249 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/1174de23e4c04248a59665348fee54ff as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1174de23e4c04248a59665348fee54ff 2024-11-13T18:37:12,255 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/B of 4f01271d10fc9cb8342007ba2bfde68b into 1174de23e4c04248a59665348fee54ff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:12,256 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:12,256 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/B, priority=13, startTime=1731523032202; duration=0sec 2024-11-13T18:37:12,256 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:12,256 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:B 2024-11-13T18:37:12,256 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:12,265 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:12,265 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 4f01271d10fc9cb8342007ba2bfde68b/C is initiating minor compaction (all files) 2024-11-13T18:37:12,266 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4f01271d10fc9cb8342007ba2bfde68b/C in TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:12,266 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/d9640928c6db4945a325601ea360af2b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/40bb6fe404514216aefb51c3110742cd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6952fe3b423b44e6a1caf5ec2a0f093f] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp, totalSize=36.4 K 2024-11-13T18:37:12,270 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting d9640928c6db4945a325601ea360af2b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731523026658 2024-11-13T18:37:12,271 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 40bb6fe404514216aefb51c3110742cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731523027886 2024-11-13T18:37:12,272 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6952fe3b423b44e6a1caf5ec2a0f093f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731523030150 2024-11-13T18:37:12,283 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:12,284 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/c2c1b785a757461f853e33ea93a1fa02 is 50, key is test_row_0/C:col10/1731523030808/Put/seqid=0 2024-11-13T18:37:12,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742015_1191 (size=12983) 2024-11-13T18:37:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T18:37:12,334 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:12,650 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4f01271d10fc9cb8342007ba2bfde68b#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:12,651 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/f0b9ab6bc80b45aba7708d9761e87ce9 is 175, key is test_row_0/A:col10/1731523030808/Put/seqid=0 2024-11-13T18:37:12,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742016_1192 (size=31937) 2024-11-13T18:37:12,706 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/c2c1b785a757461f853e33ea93a1fa02 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/c2c1b785a757461f853e33ea93a1fa02 2024-11-13T18:37:12,712 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/C of 4f01271d10fc9cb8342007ba2bfde68b into c2c1b785a757461f853e33ea93a1fa02(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:12,712 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:12,712 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/C, priority=13, startTime=1731523032202; duration=0sec 2024-11-13T18:37:12,713 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:12,713 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:C 2024-11-13T18:37:13,016 INFO [Thread-574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-574. 2024-11-13T18:37:13,016 DEBUG [Thread-574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:13,016 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:13,016 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:13,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:13,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:13,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:13,025 INFO [Thread-566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-566. 2024-11-13T18:37:13,025 DEBUG [Thread-566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:13,025 DEBUG [Thread-566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:13,026 DEBUG [Thread-566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:13,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:13,035 INFO [Thread-570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-570. 2024-11-13T18:37:13,036 DEBUG [Thread-570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:13,036 DEBUG [Thread-570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:13,036 DEBUG [Thread-570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:13,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:13,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113de37881cfc724241b0d8d36c1b9b3f33_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523030829/Put/seqid=0 2024-11-13T18:37:13,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742017_1193 (size=12454) 2024-11-13T18:37:13,102 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/f0b9ab6bc80b45aba7708d9761e87ce9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f0b9ab6bc80b45aba7708d9761e87ce9 2024-11-13T18:37:13,119 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4f01271d10fc9cb8342007ba2bfde68b/A of 4f01271d10fc9cb8342007ba2bfde68b into f0b9ab6bc80b45aba7708d9761e87ce9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:13,119 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:13,119 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., storeName=4f01271d10fc9cb8342007ba2bfde68b/A, priority=13, startTime=1731523032201; duration=0sec 2024-11-13T18:37:13,119 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:13,119 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4f01271d10fc9cb8342007ba2bfde68b:A 2024-11-13T18:37:13,466 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:13,474 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113de37881cfc724241b0d8d36c1b9b3f33_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de37881cfc724241b0d8d36c1b9b3f33_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:13,477 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/426024cc53be4d69b36ca3b888b97526, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:13,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/426024cc53be4d69b36ca3b888b97526 is 175, key is test_row_0/A:col10/1731523030829/Put/seqid=0 2024-11-13T18:37:13,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742018_1194 (size=31255) 2024-11-13T18:37:13,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:37:13,902 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/426024cc53be4d69b36ca3b888b97526 2024-11-13T18:37:13,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/0a488f397abd4c47adaed141bccae01a is 50, key is test_row_0/B:col10/1731523030829/Put/seqid=0 2024-11-13T18:37:13,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742019_1195 (size=12301) 2024-11-13T18:37:13,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/0a488f397abd4c47adaed141bccae01a 2024-11-13T18:37:13,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/2373328ca61b4141a1f19295eb7c7e27 is 50, key is test_row_0/C:col10/1731523030829/Put/seqid=0 2024-11-13T18:37:14,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742020_1196 (size=12301) 2024-11-13T18:37:14,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/2373328ca61b4141a1f19295eb7c7e27 2024-11-13T18:37:14,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/426024cc53be4d69b36ca3b888b97526 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/426024cc53be4d69b36ca3b888b97526 2024-11-13T18:37:14,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/426024cc53be4d69b36ca3b888b97526, entries=150, sequenceid=303, filesize=30.5 K 2024-11-13T18:37:14,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/0a488f397abd4c47adaed141bccae01a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/0a488f397abd4c47adaed141bccae01a 2024-11-13T18:37:14,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/0a488f397abd4c47adaed141bccae01a, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T18:37:14,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/2373328ca61b4141a1f19295eb7c7e27 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2373328ca61b4141a1f19295eb7c7e27 2024-11-13T18:37:14,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2373328ca61b4141a1f19295eb7c7e27, entries=150, sequenceid=303, filesize=12.0 K 2024-11-13T18:37:14,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=6.71 KB/6870 for 4f01271d10fc9cb8342007ba2bfde68b in 1648ms, sequenceid=303, compaction requested=false 2024-11-13T18:37:14,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4f01271d10fc9cb8342007ba2bfde68b: 2024-11-13T18:37:21,244 INFO [Thread-568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-568. 2024-11-13T18:37:21,245 DEBUG [Thread-568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:21,245 DEBUG [Thread-568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:21,245 DEBUG [Thread-568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:21,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:21,275 INFO [Thread-572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-572. 2024-11-13T18:37:21,276 DEBUG [Thread-572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:21,276 DEBUG [Thread-572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:21,276 DEBUG [Thread-572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:21,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1118 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1109 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 534 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1602 rows 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 537 2024-11-13T18:37:21,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1610 rows 2024-11-13T18:37:21,277 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:37:21,277 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:37:21,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:21,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:21,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:21,278 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:37:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:37:21,282 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523041282"}]},"ts":"1731523041282"} 2024-11-13T18:37:21,284 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:37:21,284 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:37:21,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:37:21,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, UNASSIGN}] 2024-11-13T18:37:21,287 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, UNASSIGN 2024-11-13T18:37:21,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=4f01271d10fc9cb8342007ba2bfde68b, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:21,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, UNASSIGN because future has completed 2024-11-13T18:37:21,289 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:37:21,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:37:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:37:21,449 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:21,450 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:37:21,450 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 4f01271d10fc9cb8342007ba2bfde68b, disabling compactions & flushes 2024-11-13T18:37:21,450 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:21,450 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:21,450 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. after waiting 0 ms 2024-11-13T18:37:21,450 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:21,450 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 4f01271d10fc9cb8342007ba2bfde68b 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-13T18:37:21,462 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111357359536d9ed4cd8b7796bf987fcd48d_4f01271d10fc9cb8342007ba2bfde68b is 50, key is test_row_0/A:col10/1731523041274/Put/seqid=0 2024-11-13T18:37:21,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742021_1197 (size=7374) 2024-11-13T18:37:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:37:21,902 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:37:21,908 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111357359536d9ed4cd8b7796bf987fcd48d_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111357359536d9ed4cd8b7796bf987fcd48d_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:21,909 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/e0db05fb7d7e4427bdcbdd516d9189e6, store: [table=TestAcidGuarantees family=A region=4f01271d10fc9cb8342007ba2bfde68b] 2024-11-13T18:37:21,910 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/e0db05fb7d7e4427bdcbdd516d9189e6 is 175, key is test_row_0/A:col10/1731523041274/Put/seqid=0 2024-11-13T18:37:21,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742022_1198 (size=13865) 2024-11-13T18:37:21,930 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/e0db05fb7d7e4427bdcbdd516d9189e6 2024-11-13T18:37:21,956 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/80a6884e3d0b4a31af6650e59cfb460f is 50, key is test_row_0/B:col10/1731523041274/Put/seqid=0 2024-11-13T18:37:21,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742023_1199 (size=7415) 2024-11-13T18:37:22,389 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/80a6884e3d0b4a31af6650e59cfb460f 2024-11-13T18:37:22,404 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/58e334b3c987478880d1d89e3cfde147 is 50, key is test_row_0/C:col10/1731523041274/Put/seqid=0 2024-11-13T18:37:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:37:22,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742024_1200 (size=7415) 2024-11-13T18:37:22,816 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/58e334b3c987478880d1d89e3cfde147 2024-11-13T18:37:22,822 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/A/e0db05fb7d7e4427bdcbdd516d9189e6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/e0db05fb7d7e4427bdcbdd516d9189e6 2024-11-13T18:37:22,828 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/e0db05fb7d7e4427bdcbdd516d9189e6, entries=50, sequenceid=310, filesize=13.5 K 2024-11-13T18:37:22,829 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/B/80a6884e3d0b4a31af6650e59cfb460f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/80a6884e3d0b4a31af6650e59cfb460f 2024-11-13T18:37:22,837 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/80a6884e3d0b4a31af6650e59cfb460f, entries=50, sequenceid=310, filesize=7.2 K 2024-11-13T18:37:22,839 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/.tmp/C/58e334b3c987478880d1d89e3cfde147 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/58e334b3c987478880d1d89e3cfde147 2024-11-13T18:37:22,845 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/58e334b3c987478880d1d89e3cfde147, entries=50, sequenceid=310, filesize=7.2 K 2024-11-13T18:37:22,846 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 4f01271d10fc9cb8342007ba2bfde68b in 1396ms, sequenceid=310, compaction requested=true 2024-11-13T18:37:22,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9bc9d1980d2047aab2cb76e20740e146, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/614db059e5794df48e56666e856a8dce, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/87f4e1ea471e4b6385ceb46a4f51afad, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/53a7fba66caa4b69a280fd4f65163a71, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/b9b6d1ba592b4095a4c3abc79916ae76, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/11bbb39504f84e82ad07090676ef732e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/5f77d7637d1749c2b62d89dfa2d3ae96, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/3463b0d024fd4260a61abcd5b3299a24, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60676741d41a4f6f9716c4c65cea6ca3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0b686fec64484904b59d882205964981, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/185236dcad9f473c83ac7b8e0d634ad2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9808d6314b85404b8b8c9e23909cae84, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/cf7ae74c4f08484a9646527a0a04debc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/45207759ae8d4be2b1abdd1f81bbee96, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/a7d591727b4f4c14843cf4e56459c69f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/c232e94063e847bb8065a925e2d7fef0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/2570c955725c4274bafc8289e3e3efc0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60a27beb00d54157bf8585734d2e74c7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0907a75e83cd47ac8ae0e5b14cb288f1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f7890f8c784840e9b1aa59a47d87fd93] to archive 2024-11-13T18:37:22,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:37:22,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9bc9d1980d2047aab2cb76e20740e146 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9bc9d1980d2047aab2cb76e20740e146 2024-11-13T18:37:22,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/614db059e5794df48e56666e856a8dce to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/614db059e5794df48e56666e856a8dce 2024-11-13T18:37:22,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/87f4e1ea471e4b6385ceb46a4f51afad to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/87f4e1ea471e4b6385ceb46a4f51afad 2024-11-13T18:37:22,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/53a7fba66caa4b69a280fd4f65163a71 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/53a7fba66caa4b69a280fd4f65163a71 2024-11-13T18:37:22,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/b9b6d1ba592b4095a4c3abc79916ae76 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/b9b6d1ba592b4095a4c3abc79916ae76 2024-11-13T18:37:22,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/11bbb39504f84e82ad07090676ef732e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/11bbb39504f84e82ad07090676ef732e 2024-11-13T18:37:22,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/5f77d7637d1749c2b62d89dfa2d3ae96 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/5f77d7637d1749c2b62d89dfa2d3ae96 2024-11-13T18:37:22,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/3463b0d024fd4260a61abcd5b3299a24 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/3463b0d024fd4260a61abcd5b3299a24 2024-11-13T18:37:22,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60676741d41a4f6f9716c4c65cea6ca3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60676741d41a4f6f9716c4c65cea6ca3 2024-11-13T18:37:22,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0b686fec64484904b59d882205964981 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0b686fec64484904b59d882205964981 2024-11-13T18:37:22,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/185236dcad9f473c83ac7b8e0d634ad2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/185236dcad9f473c83ac7b8e0d634ad2 2024-11-13T18:37:22,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9808d6314b85404b8b8c9e23909cae84 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/9808d6314b85404b8b8c9e23909cae84 2024-11-13T18:37:22,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/cf7ae74c4f08484a9646527a0a04debc to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/cf7ae74c4f08484a9646527a0a04debc 2024-11-13T18:37:22,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/45207759ae8d4be2b1abdd1f81bbee96 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/45207759ae8d4be2b1abdd1f81bbee96 2024-11-13T18:37:22,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/a7d591727b4f4c14843cf4e56459c69f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/a7d591727b4f4c14843cf4e56459c69f 2024-11-13T18:37:22,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/c232e94063e847bb8065a925e2d7fef0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/c232e94063e847bb8065a925e2d7fef0 2024-11-13T18:37:22,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/2570c955725c4274bafc8289e3e3efc0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/2570c955725c4274bafc8289e3e3efc0 2024-11-13T18:37:22,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60a27beb00d54157bf8585734d2e74c7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/60a27beb00d54157bf8585734d2e74c7 2024-11-13T18:37:22,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0907a75e83cd47ac8ae0e5b14cb288f1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/0907a75e83cd47ac8ae0e5b14cb288f1 2024-11-13T18:37:22,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f7890f8c784840e9b1aa59a47d87fd93 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f7890f8c784840e9b1aa59a47d87fd93 2024-11-13T18:37:22,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/21bc13ce5be84398b159d7a628cce65e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/a831a8d1ef2446ec9994ddf72cada0ea, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3162e0b683aa45f4a49ae480f9f0071d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/12e39c83cc5440b98c44ce10caf60559, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/d73528422e904b348833d6c99283f5b2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fc41bd5fc72f4e55ae5e2db40c78ede0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/17e6d70d4713453dbad99f9f2e7f027b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/5890bc6974d64c63b13ac2be63230df1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fd4e69d1206348b09c9fff0e8fb59459, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/6d6d5bbed7d24e06acbb8aedba536ab8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/2abcfb9b5eeb40e281cda5b2890d87d0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1501a3b7ce0a4f659af4188afe05962b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3e63fbeef50a4f1aa1b9f206b5c128b3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/df19b158b78a444f92cdd8f9b0dfaedb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/97992b7bab414d9fa711e4b7d73b1913, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/e65ad04750b9440589badffbd77a2ee0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/62964ca9770240aa83c9757a81435e6c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/9fdb77fdae5648f5bff7b285dcdf458d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/50538dc8aff148a6b5eeff0dc29b5ca7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/f056d3dd69de4f25900d78f70d9b4a30] to archive 2024-11-13T18:37:22,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:37:22,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/21bc13ce5be84398b159d7a628cce65e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/21bc13ce5be84398b159d7a628cce65e 2024-11-13T18:37:22,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/a831a8d1ef2446ec9994ddf72cada0ea to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/a831a8d1ef2446ec9994ddf72cada0ea 2024-11-13T18:37:22,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3162e0b683aa45f4a49ae480f9f0071d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3162e0b683aa45f4a49ae480f9f0071d 2024-11-13T18:37:22,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/12e39c83cc5440b98c44ce10caf60559 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/12e39c83cc5440b98c44ce10caf60559 2024-11-13T18:37:22,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/d73528422e904b348833d6c99283f5b2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/d73528422e904b348833d6c99283f5b2 2024-11-13T18:37:22,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fc41bd5fc72f4e55ae5e2db40c78ede0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fc41bd5fc72f4e55ae5e2db40c78ede0 2024-11-13T18:37:22,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/17e6d70d4713453dbad99f9f2e7f027b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/17e6d70d4713453dbad99f9f2e7f027b 2024-11-13T18:37:22,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/5890bc6974d64c63b13ac2be63230df1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/5890bc6974d64c63b13ac2be63230df1 2024-11-13T18:37:22,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fd4e69d1206348b09c9fff0e8fb59459 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/fd4e69d1206348b09c9fff0e8fb59459 2024-11-13T18:37:22,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/6d6d5bbed7d24e06acbb8aedba536ab8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/6d6d5bbed7d24e06acbb8aedba536ab8 2024-11-13T18:37:22,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/2abcfb9b5eeb40e281cda5b2890d87d0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/2abcfb9b5eeb40e281cda5b2890d87d0 2024-11-13T18:37:22,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1501a3b7ce0a4f659af4188afe05962b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1501a3b7ce0a4f659af4188afe05962b 2024-11-13T18:37:22,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3e63fbeef50a4f1aa1b9f206b5c128b3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/3e63fbeef50a4f1aa1b9f206b5c128b3 2024-11-13T18:37:22,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/df19b158b78a444f92cdd8f9b0dfaedb to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/df19b158b78a444f92cdd8f9b0dfaedb 2024-11-13T18:37:22,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/97992b7bab414d9fa711e4b7d73b1913 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/97992b7bab414d9fa711e4b7d73b1913 2024-11-13T18:37:22,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/e65ad04750b9440589badffbd77a2ee0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/e65ad04750b9440589badffbd77a2ee0 2024-11-13T18:37:22,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/62964ca9770240aa83c9757a81435e6c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/62964ca9770240aa83c9757a81435e6c 2024-11-13T18:37:22,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/9fdb77fdae5648f5bff7b285dcdf458d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/9fdb77fdae5648f5bff7b285dcdf458d 2024-11-13T18:37:22,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/50538dc8aff148a6b5eeff0dc29b5ca7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/50538dc8aff148a6b5eeff0dc29b5ca7 2024-11-13T18:37:22,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/f056d3dd69de4f25900d78f70d9b4a30 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/f056d3dd69de4f25900d78f70d9b4a30 2024-11-13T18:37:22,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e4c1fb41a9024869bf58aa4fb1e1eddb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/390948f484994adda1b254165e6a7370, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e74c1194f59d49a09efd234aaec117c1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1f4d4213fe2544e6accccd57c9af4f18, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/182a7b2344df480bad70b8f9ba3aa75a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6eea27e7601f46f9af4690b283cd8d22, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/7c6ca5289bc5466e9c8a78e8ffc90018, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a1f530ee16b342f9a07f5e15137aa532, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8993a98644fa43548568672af03dd0e1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a38da16c22bf48d5b1055d6e341695d5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1066b2e746b945ac86abcd9219434a79, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2eaf6ba69442462da752f81fea162e6d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8a24d89cbf4d40ba940f9688ec3d9f10, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/bdae0cdca7c248a4af98f87bdd4e536c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/529a802a4c5542a7b8903c7f7cfcc4a5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/0a34b2e6f6f24adc857622e984c4b49f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/d9640928c6db4945a325601ea360af2b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/457c880ebfeb4c628a61cd3ddadbac0e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/40bb6fe404514216aefb51c3110742cd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6952fe3b423b44e6a1caf5ec2a0f093f] to archive 2024-11-13T18:37:22,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:37:22,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e4c1fb41a9024869bf58aa4fb1e1eddb to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e4c1fb41a9024869bf58aa4fb1e1eddb 2024-11-13T18:37:22,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/390948f484994adda1b254165e6a7370 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/390948f484994adda1b254165e6a7370 2024-11-13T18:37:22,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e74c1194f59d49a09efd234aaec117c1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/e74c1194f59d49a09efd234aaec117c1 2024-11-13T18:37:22,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1f4d4213fe2544e6accccd57c9af4f18 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1f4d4213fe2544e6accccd57c9af4f18 2024-11-13T18:37:22,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/182a7b2344df480bad70b8f9ba3aa75a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/182a7b2344df480bad70b8f9ba3aa75a 2024-11-13T18:37:22,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6eea27e7601f46f9af4690b283cd8d22 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6eea27e7601f46f9af4690b283cd8d22 2024-11-13T18:37:22,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/7c6ca5289bc5466e9c8a78e8ffc90018 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/7c6ca5289bc5466e9c8a78e8ffc90018 2024-11-13T18:37:22,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a1f530ee16b342f9a07f5e15137aa532 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a1f530ee16b342f9a07f5e15137aa532 2024-11-13T18:37:22,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8993a98644fa43548568672af03dd0e1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8993a98644fa43548568672af03dd0e1 2024-11-13T18:37:22,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a38da16c22bf48d5b1055d6e341695d5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/a38da16c22bf48d5b1055d6e341695d5 2024-11-13T18:37:22,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1066b2e746b945ac86abcd9219434a79 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/1066b2e746b945ac86abcd9219434a79 2024-11-13T18:37:22,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2eaf6ba69442462da752f81fea162e6d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2eaf6ba69442462da752f81fea162e6d 2024-11-13T18:37:22,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8a24d89cbf4d40ba940f9688ec3d9f10 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/8a24d89cbf4d40ba940f9688ec3d9f10 2024-11-13T18:37:22,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/bdae0cdca7c248a4af98f87bdd4e536c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/bdae0cdca7c248a4af98f87bdd4e536c 2024-11-13T18:37:22,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/529a802a4c5542a7b8903c7f7cfcc4a5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/529a802a4c5542a7b8903c7f7cfcc4a5 2024-11-13T18:37:22,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/0a34b2e6f6f24adc857622e984c4b49f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/0a34b2e6f6f24adc857622e984c4b49f 2024-11-13T18:37:22,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/d9640928c6db4945a325601ea360af2b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/d9640928c6db4945a325601ea360af2b 2024-11-13T18:37:22,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/457c880ebfeb4c628a61cd3ddadbac0e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/457c880ebfeb4c628a61cd3ddadbac0e 2024-11-13T18:37:22,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/40bb6fe404514216aefb51c3110742cd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/40bb6fe404514216aefb51c3110742cd 2024-11-13T18:37:22,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6952fe3b423b44e6a1caf5ec2a0f093f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/6952fe3b423b44e6a1caf5ec2a0f093f 2024-11-13T18:37:22,999 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/recovered.edits/313.seqid, newMaxSeqId=313, maxSeqId=4 2024-11-13T18:37:23,000 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. 2024-11-13T18:37:23,000 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 4f01271d10fc9cb8342007ba2bfde68b: Waiting for close lock at 1731523041450Running coprocessor pre-close hooks at 1731523041450Disabling compacts and flushes for region at 1731523041450Disabling writes for close at 1731523041450Obtaining lock to block concurrent updates at 1731523041450Preparing flush snapshotting stores in 4f01271d10fc9cb8342007ba2bfde68b at 1731523041450Finished memstore snapshotting TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731523041451 (+1 ms)Flushing stores of TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b. at 1731523041453 (+2 ms)Flushing 4f01271d10fc9cb8342007ba2bfde68b/A: creating writer at 1731523041453Flushing mob file 4f01271d10fc9cb8342007ba2bfde68b/A: appending metadata at 1731523041462 (+9 ms)Flushing mob file 4f01271d10fc9cb8342007ba2bfde68b/A: closing flushed file at 1731523041462Flushing 4f01271d10fc9cb8342007ba2bfde68b/A: appending metadata at 1731523041909 (+447 ms)Flushing 4f01271d10fc9cb8342007ba2bfde68b/A: closing flushed file at 1731523041910 (+1 ms)Flushing 4f01271d10fc9cb8342007ba2bfde68b/B: creating writer at 1731523041946 (+36 ms)Flushing 4f01271d10fc9cb8342007ba2bfde68b/B: appending metadata at 1731523041955 (+9 ms)Flushing 4f01271d10fc9cb8342007ba2bfde68b/B: closing flushed file at 1731523041955Flushing 4f01271d10fc9cb8342007ba2bfde68b/C: creating writer at 1731523042396 (+441 ms)Flushing 4f01271d10fc9cb8342007ba2bfde68b/C: appending metadata at 1731523042403 (+7 ms)Flushing 4f01271d10fc9cb8342007ba2bfde68b/C: closing flushed file at 1731523042403Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16aa2000: reopening flushed file at 1731523042821 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dccdd87: reopening flushed file at 1731523042828 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48707b6d: reopening flushed file at 1731523042837 (+9 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 4f01271d10fc9cb8342007ba2bfde68b in 1396ms, sequenceid=310, compaction requested=true at 1731523042846 (+9 ms)Writing region close event to WAL at 1731523042987 (+141 ms)Running coprocessor post-close hooks at 1731523042999 (+12 ms)Closed at 1731523042999 2024-11-13T18:37:23,003 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,003 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=4f01271d10fc9cb8342007ba2bfde68b, regionState=CLOSED 2024-11-13T18:37:23,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:37:23,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-13T18:37:23,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 4f01271d10fc9cb8342007ba2bfde68b, server=39e84130bbc9,35235,1731522975587 in 1.7190 sec 2024-11-13T18:37:23,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-13T18:37:23,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4f01271d10fc9cb8342007ba2bfde68b, UNASSIGN in 1.7330 sec 2024-11-13T18:37:23,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T18:37:23,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7390 sec 2024-11-13T18:37:23,026 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523043026"}]},"ts":"1731523043026"} 2024-11-13T18:37:23,028 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:37:23,028 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:37:23,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7500 sec 2024-11-13T18:37:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T18:37:23,433 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:23,434 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:37:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:23,436 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T18:37:23,436 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:23,439 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,442 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/recovered.edits] 2024-11-13T18:37:23,445 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/426024cc53be4d69b36ca3b888b97526 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/426024cc53be4d69b36ca3b888b97526 2024-11-13T18:37:23,446 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/e0db05fb7d7e4427bdcbdd516d9189e6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/e0db05fb7d7e4427bdcbdd516d9189e6 2024-11-13T18:37:23,448 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f0b9ab6bc80b45aba7708d9761e87ce9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/A/f0b9ab6bc80b45aba7708d9761e87ce9 2024-11-13T18:37:23,450 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/0a488f397abd4c47adaed141bccae01a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/0a488f397abd4c47adaed141bccae01a 2024-11-13T18:37:23,452 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1174de23e4c04248a59665348fee54ff to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/1174de23e4c04248a59665348fee54ff 2024-11-13T18:37:23,453 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/80a6884e3d0b4a31af6650e59cfb460f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/B/80a6884e3d0b4a31af6650e59cfb460f 2024-11-13T18:37:23,456 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2373328ca61b4141a1f19295eb7c7e27 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/2373328ca61b4141a1f19295eb7c7e27 2024-11-13T18:37:23,457 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/58e334b3c987478880d1d89e3cfde147 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/58e334b3c987478880d1d89e3cfde147 2024-11-13T18:37:23,459 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/c2c1b785a757461f853e33ea93a1fa02 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/C/c2c1b785a757461f853e33ea93a1fa02 2024-11-13T18:37:23,462 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/recovered.edits/313.seqid to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b/recovered.edits/313.seqid 2024-11-13T18:37:23,463 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,463 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:37:23,464 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:37:23,465 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:37:23,469 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302e1efc373264a78b96e73f524222cca_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302e1efc373264a78b96e73f524222cca_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,470 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132bedc91df1e54bb5bbc1e45181aeabe2_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132bedc91df1e54bb5bbc1e45181aeabe2_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,472 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c1fda2d96a74725a103f0e448764a81_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132c1fda2d96a74725a103f0e448764a81_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,473 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113374dea615e6f44fbb9b672be86561ea5_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113374dea615e6f44fbb9b672be86561ea5_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,474 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111339e6aa6fb0c94969aef6cadaa2c8c0e7_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111339e6aa6fb0c94969aef6cadaa2c8c0e7_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,476 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133d4539fbe18749a686932de459737e96_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133d4539fbe18749a686932de459737e96_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,477 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347215677a9694327a41dce5f48191f71_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347215677a9694327a41dce5f48191f71_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,478 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134d1bcac0eba048ec829b4c919d492320_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134d1bcac0eba048ec829b4c919d492320_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,479 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135332555191c846d3bd91ed6033071303_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135332555191c846d3bd91ed6033071303_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,480 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111357359536d9ed4cd8b7796bf987fcd48d_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111357359536d9ed4cd8b7796bf987fcd48d_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,481 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359d006bbc3d3490997b77f88daff461c_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359d006bbc3d3490997b77f88daff461c_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,483 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111369be22121eeb46639f0c791ae9a81018_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111369be22121eeb46639f0c791ae9a81018_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,484 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371caa8339583413785c42db3657d92e4_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371caa8339583413785c42db3657d92e4_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,485 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137cf342aac31448fbbb24d7e105beb42e_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137cf342aac31448fbbb24d7e105beb42e_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,486 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113979c7fe94f184d82b83c4feed0724860_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113979c7fe94f184d82b83c4feed0724860_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,487 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de37881cfc724241b0d8d36c1b9b3f33_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de37881cfc724241b0d8d36c1b9b3f33_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,488 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e9a6a2fa672647e79e46f18dedaa0e7b_4f01271d10fc9cb8342007ba2bfde68b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e9a6a2fa672647e79e46f18dedaa0e7b_4f01271d10fc9cb8342007ba2bfde68b 2024-11-13T18:37:23,489 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:37:23,491 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:23,494 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:37:23,498 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:37:23,499 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:23,499 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:37:23,499 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731523043499"}]},"ts":"9223372036854775807"} 2024-11-13T18:37:23,501 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:37:23,501 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4f01271d10fc9cb8342007ba2bfde68b, NAME => 'TestAcidGuarantees,,1731523006514.4f01271d10fc9cb8342007ba2bfde68b.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:37:23,501 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:37:23,502 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731523043501"}]},"ts":"9223372036854775807"} 2024-11-13T18:37:23,503 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:37:23,504 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:23,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-13T18:37:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T18:37:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:37:23,544 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:23,555 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 240), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=2705 (was 2312) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3800 (was 3204) - AvailableMemoryMB LEAK? - 2024-11-13T18:37:23,567 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=2705, ProcessCount=11, AvailableMemoryMB=3799 2024-11-13T18:37:23,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:37:23,569 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:37:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:23,572 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:37:23,572 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:23,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-13T18:37:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-13T18:37:23,577 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:37:23,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742025_1201 (size=959) 2024-11-13T18:37:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-13T18:37:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-13T18:37:23,989 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 2024-11-13T18:37:23,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742026_1202 (size=53) 2024-11-13T18:37:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-13T18:37:24,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:37:24,400 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f8326d4028e020fa48ab6da5484ce85e, disabling compactions & flushes 2024-11-13T18:37:24,400 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:24,400 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:24,400 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. after waiting 0 ms 2024-11-13T18:37:24,400 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:24,400 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:24,400 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8326d4028e020fa48ab6da5484ce85e: Waiting for close lock at 1731523044399Disabling compacts and flushes for region at 1731523044399Disabling writes for close at 1731523044400 (+1 ms)Writing region close event to WAL at 1731523044400Closed at 1731523044400 2024-11-13T18:37:24,401 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:37:24,401 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731523044401"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731523044401"}]},"ts":"1731523044401"} 2024-11-13T18:37:24,403 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:37:24,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:37:24,404 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523044404"}]},"ts":"1731523044404"} 2024-11-13T18:37:24,405 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:37:24,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, ASSIGN}] 2024-11-13T18:37:24,406 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, ASSIGN 2024-11-13T18:37:24,407 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, ASSIGN; state=OFFLINE, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=false 2024-11-13T18:37:24,558 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f8326d4028e020fa48ab6da5484ce85e, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:24,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, ASSIGN because future has completed 2024-11-13T18:37:24,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:37:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-13T18:37:24,719 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:24,719 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:37:24,720 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,720 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:37:24,720 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,720 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,722 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,723 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8326d4028e020fa48ab6da5484ce85e columnFamilyName A 2024-11-13T18:37:24,723 DEBUG [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:24,724 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] regionserver.HStore(327): Store=f8326d4028e020fa48ab6da5484ce85e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:24,724 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,725 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8326d4028e020fa48ab6da5484ce85e columnFamilyName B 2024-11-13T18:37:24,725 DEBUG [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:24,725 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] regionserver.HStore(327): Store=f8326d4028e020fa48ab6da5484ce85e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:24,725 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,727 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8326d4028e020fa48ab6da5484ce85e columnFamilyName C 2024-11-13T18:37:24,727 DEBUG [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:24,727 INFO [StoreOpener-f8326d4028e020fa48ab6da5484ce85e-1 {}] regionserver.HStore(327): Store=f8326d4028e020fa48ab6da5484ce85e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:24,727 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,728 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,728 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,729 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,729 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,730 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:37:24,731 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,734 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:37:24,735 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened f8326d4028e020fa48ab6da5484ce85e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75251719, jitterRate=0.12133799493312836}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:37:24,735 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:24,735 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for f8326d4028e020fa48ab6da5484ce85e: Running coprocessor pre-open hook at 1731523044720Writing region info on filesystem at 1731523044720Initializing all the Stores at 1731523044721 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523044721Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523044721Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523044721Cleaning up temporary data from old regions at 1731523044729 (+8 ms)Running coprocessor post-open hooks at 1731523044735 (+6 ms)Region opened successfully at 1731523044735 2024-11-13T18:37:24,736 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., pid=55, masterSystemTime=1731523044715 2024-11-13T18:37:24,738 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:24,738 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:24,739 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f8326d4028e020fa48ab6da5484ce85e, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:24,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:37:24,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T18:37:24,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 in 182 msec 2024-11-13T18:37:24,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-13T18:37:24,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, ASSIGN in 342 msec 2024-11-13T18:37:24,749 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:37:24,750 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523044749"}]},"ts":"1731523044749"} 2024-11-13T18:37:24,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:37:24,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:37:24,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-13T18:37:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-13T18:37:25,734 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:25,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,736 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177a0e4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,738 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c90b409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:37:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 39e84130bbc9,36105,1731522974847 2024-11-13T18:37:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a04f4fc 2024-11-13T18:37:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:37:25,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:37:25,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,746 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7f88a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,747 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0c1c24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,749 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@561af451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,750 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a725c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,752 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56763ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d0f3ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,755 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713c07e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,757 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb058e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,759 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e02922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,760 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c3fb37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,762 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,763 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd9afce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2798b005, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536bbf10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,768 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a275ef3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,769 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231255a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,771 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4653e1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,772 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f206d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:25,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:25,773 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2f7d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:25,775 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6813a660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:25,776 DEBUG [Thread-982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,776 DEBUG [Thread-980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,776 DEBUG [Thread-969 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,776 DEBUG [Thread-986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,776 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:25,776 DEBUG [Thread-977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,776 DEBUG [Thread-971 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,777 DEBUG [Thread-973 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,777 DEBUG [Thread-988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-13T18:37:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,778 DEBUG [Thread-975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,781 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:37:25,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,786 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:25,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,792 DEBUG [Thread-984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:25,795 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:37:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:25,798 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:25,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:25,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/308a9c4773b74187ac2c3b49db46bb73 is 50, key is test_row_0/A:col10/1731523045798/Put/seqid=0 2024-11-13T18:37:25,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742027_1203 (size=14341) 2024-11-13T18:37:25,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523105832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523105835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523105839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523105842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523105848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/308a9c4773b74187ac2c3b49db46bb73 2024-11-13T18:37:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:37:25,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0fb2f663670b4f55a65a66fea49d932f is 50, key is test_row_0/B:col10/1731523045798/Put/seqid=0 2024-11-13T18:37:25,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742028_1204 (size=12001) 2024-11-13T18:37:25,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0fb2f663670b4f55a65a66fea49d932f 2024-11-13T18:37:25,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:37:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:25,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:25,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523105954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523105955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523105955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523105957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523105964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:25,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/991567b1a37649cc82c3c7fb36f5935b is 50, key is test_row_0/C:col10/1731523045798/Put/seqid=0 2024-11-13T18:37:26,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742029_1205 (size=12001) 2024-11-13T18:37:26,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/991567b1a37649cc82c3c7fb36f5935b 2024-11-13T18:37:26,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/308a9c4773b74187ac2c3b49db46bb73 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/308a9c4773b74187ac2c3b49db46bb73 2024-11-13T18:37:26,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/308a9c4773b74187ac2c3b49db46bb73, entries=200, sequenceid=15, filesize=14.0 K 2024-11-13T18:37:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0fb2f663670b4f55a65a66fea49d932f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fb2f663670b4f55a65a66fea49d932f 2024-11-13T18:37:26,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fb2f663670b4f55a65a66fea49d932f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T18:37:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/991567b1a37649cc82c3c7fb36f5935b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/991567b1a37649cc82c3c7fb36f5935b 2024-11-13T18:37:26,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/991567b1a37649cc82c3c7fb36f5935b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T18:37:26,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f8326d4028e020fa48ab6da5484ce85e in 292ms, sequenceid=15, compaction requested=false 2024-11-13T18:37:26,090 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T18:37:26,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:37:26,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T18:37:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:26,109 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T18:37:26,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ecfb60e1540c49b391d94bb06094b388 is 50, key is test_row_0/A:col10/1731523045838/Put/seqid=0 2024-11-13T18:37:26,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742030_1206 (size=12001) 2024-11-13T18:37:26,173 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ecfb60e1540c49b391d94bb06094b388 2024-11-13T18:37:26,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:26,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/c358bb5d60264c8b9ffc5ba654a6cff1 is 50, key is test_row_0/B:col10/1731523045838/Put/seqid=0 2024-11-13T18:37:26,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523106199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523106195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523106203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523106213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523106214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742031_1207 (size=12001) 2024-11-13T18:37:26,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523106314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523106314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523106314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523106327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523106327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:37:26,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523106523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523106523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523106524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523106534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523106534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,619 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/c358bb5d60264c8b9ffc5ba654a6cff1 2024-11-13T18:37:26,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/78a4556b113a412d8b1611b66a49799a is 50, key is test_row_0/C:col10/1731523045838/Put/seqid=0 2024-11-13T18:37:26,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742032_1208 (size=12001) 2024-11-13T18:37:26,708 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/78a4556b113a412d8b1611b66a49799a 2024-11-13T18:37:26,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ecfb60e1540c49b391d94bb06094b388 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ecfb60e1540c49b391d94bb06094b388 2024-11-13T18:37:26,720 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ecfb60e1540c49b391d94bb06094b388, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T18:37:26,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/c358bb5d60264c8b9ffc5ba654a6cff1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c358bb5d60264c8b9ffc5ba654a6cff1 2024-11-13T18:37:26,729 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c358bb5d60264c8b9ffc5ba654a6cff1, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T18:37:26,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/78a4556b113a412d8b1611b66a49799a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/78a4556b113a412d8b1611b66a49799a 2024-11-13T18:37:26,737 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/78a4556b113a412d8b1611b66a49799a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T18:37:26,738 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f8326d4028e020fa48ab6da5484ce85e in 629ms, sequenceid=37, compaction requested=false 2024-11-13T18:37:26,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-13T18:37:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-13T18:37:26,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T18:37:26,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 957 msec 2024-11-13T18:37:26,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 970 msec 2024-11-13T18:37:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:26,849 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:26,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T18:37:26,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/6ac3e48a2855464a824ad7145275bdda is 50, key is test_row_0/A:col10/1731523046197/Put/seqid=0 2024-11-13T18:37:26,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742033_1209 (size=12001) 2024-11-13T18:37:26,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/6ac3e48a2855464a824ad7145275bdda 2024-11-13T18:37:26,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/84164f8284e5463a99fb1e022a441673 is 50, key is test_row_0/B:col10/1731523046197/Put/seqid=0 2024-11-13T18:37:26,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523106890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523106891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523106892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523106902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523106902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T18:37:26,914 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:26,914 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-13T18:37:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:37:26,917 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:26,918 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:26,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:26,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742034_1210 (size=12001) 2024-11-13T18:37:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523107014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523107014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523107014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523107014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523107014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:37:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:37:27,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:27,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:37:27,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:27,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523107224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523107224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523107224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523107224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:37:27,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523107235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/84164f8284e5463a99fb1e022a441673 2024-11-13T18:37:27,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/bc19d36c8dec4d1fa6a825ce62960284 is 50, key is test_row_0/C:col10/1731523046197/Put/seqid=0 2024-11-13T18:37:27,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:37:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742035_1211 (size=12001) 2024-11-13T18:37:27,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523107534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:37:27,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523107545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523107545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523107545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523107545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:27,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:37:27,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:27,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:37:27,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:27,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/bc19d36c8dec4d1fa6a825ce62960284 2024-11-13T18:37:27,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/6ac3e48a2855464a824ad7145275bdda as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6ac3e48a2855464a824ad7145275bdda 2024-11-13T18:37:27,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6ac3e48a2855464a824ad7145275bdda, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T18:37:27,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/84164f8284e5463a99fb1e022a441673 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/84164f8284e5463a99fb1e022a441673 2024-11-13T18:37:27,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:37:27,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:27,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/84164f8284e5463a99fb1e022a441673, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T18:37:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/bc19d36c8dec4d1fa6a825ce62960284 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/bc19d36c8dec4d1fa6a825ce62960284 2024-11-13T18:37:27,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/bc19d36c8dec4d1fa6a825ce62960284, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T18:37:27,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for f8326d4028e020fa48ab6da5484ce85e in 1041ms, sequenceid=54, compaction requested=true 2024-11-13T18:37:27,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:27,890 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:27,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:27,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:27,891 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:27,892 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:27,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:27,892 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:27,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:27,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:27,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:27,892 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,892 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/308a9c4773b74187ac2c3b49db46bb73, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ecfb60e1540c49b391d94bb06094b388, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6ac3e48a2855464a824ad7145275bdda] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=37.4 K 2024-11-13T18:37:27,892 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:27,893 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:27,893 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:27,893 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fb2f663670b4f55a65a66fea49d932f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c358bb5d60264c8b9ffc5ba654a6cff1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/84164f8284e5463a99fb1e022a441673] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=35.2 K 2024-11-13T18:37:27,893 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 308a9c4773b74187ac2c3b49db46bb73, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731523045792 2024-11-13T18:37:27,893 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fb2f663670b4f55a65a66fea49d932f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731523045792 2024-11-13T18:37:27,894 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecfb60e1540c49b391d94bb06094b388, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731523045828 2024-11-13T18:37:27,894 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ac3e48a2855464a824ad7145275bdda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523046197 2024-11-13T18:37:27,894 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c358bb5d60264c8b9ffc5ba654a6cff1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731523045828 2024-11-13T18:37:27,896 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 84164f8284e5463a99fb1e022a441673, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523046197 2024-11-13T18:37:27,919 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:27,919 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/01fb61d679464ab78a17c6924127139f is 50, key is test_row_0/B:col10/1731523046197/Put/seqid=0 2024-11-13T18:37:27,929 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:27,930 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/e3e25d5da6e3437091ca0dd206023b4b is 50, key is test_row_0/A:col10/1731523046197/Put/seqid=0 2024-11-13T18:37:27,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742036_1212 (size=12104) 2024-11-13T18:37:27,979 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/01fb61d679464ab78a17c6924127139f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/01fb61d679464ab78a17c6924127139f 2024-11-13T18:37:27,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742037_1213 (size=12104) 2024-11-13T18:37:27,991 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 01fb61d679464ab78a17c6924127139f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:27,991 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:27,991 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=13, startTime=1731523047891; duration=0sec 2024-11-13T18:37:27,991 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:27,992 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:27,992 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:28,006 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:28,006 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:28,006 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:28,007 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/991567b1a37649cc82c3c7fb36f5935b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/78a4556b113a412d8b1611b66a49799a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/bc19d36c8dec4d1fa6a825ce62960284] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=35.2 K 2024-11-13T18:37:28,007 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 991567b1a37649cc82c3c7fb36f5935b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731523045792 2024-11-13T18:37:28,008 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a4556b113a412d8b1611b66a49799a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731523045828 2024-11-13T18:37:28,009 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting bc19d36c8dec4d1fa6a825ce62960284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523046197 2024-11-13T18:37:28,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T18:37:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:28,021 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T18:37:28,033 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/e3e25d5da6e3437091ca0dd206023b4b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e3e25d5da6e3437091ca0dd206023b4b 2024-11-13T18:37:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/fb6631fff2024d6ea95bb8bcce78929e is 50, key is test_row_0/A:col10/1731523046899/Put/seqid=0 2024-11-13T18:37:28,047 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:28,048 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/c1ba8483b5fa418db0e9b057237700ed is 50, key is test_row_0/C:col10/1731523046197/Put/seqid=0 2024-11-13T18:37:28,048 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into e3e25d5da6e3437091ca0dd206023b4b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:28,048 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:28,048 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=13, startTime=1731523047890; duration=0sec 2024-11-13T18:37:28,048 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:28,048 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:37:28,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:28,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742039_1215 (size=12104) 2024-11-13T18:37:28,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742038_1214 (size=12001) 2024-11-13T18:37:28,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523108080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523108083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523108084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523108087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523108087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523108193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523108193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523108194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523108204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523108204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523108403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523108403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523108403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523108413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523108423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,487 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/fb6631fff2024d6ea95bb8bcce78929e 2024-11-13T18:37:28,494 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/c1ba8483b5fa418db0e9b057237700ed as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/c1ba8483b5fa418db0e9b057237700ed 2024-11-13T18:37:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9ffb4a8d927545c9a6844946567259a6 is 50, key is test_row_0/B:col10/1731523046899/Put/seqid=0 2024-11-13T18:37:28,505 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into c1ba8483b5fa418db0e9b057237700ed(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:28,505 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:28,505 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=13, startTime=1731523047892; duration=0sec 2024-11-13T18:37:28,506 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:28,506 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:28,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742040_1216 (size=12001) 2024-11-13T18:37:28,511 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9ffb4a8d927545c9a6844946567259a6 2024-11-13T18:37:28,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0fb3caed9277459cb165a7e787fd9bb9 is 50, key is test_row_0/C:col10/1731523046899/Put/seqid=0 2024-11-13T18:37:28,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742041_1217 (size=12001) 2024-11-13T18:37:28,526 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0fb3caed9277459cb165a7e787fd9bb9 2024-11-13T18:37:28,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/fb6631fff2024d6ea95bb8bcce78929e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/fb6631fff2024d6ea95bb8bcce78929e 2024-11-13T18:37:28,537 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:37:28,542 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/fb6631fff2024d6ea95bb8bcce78929e, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T18:37:28,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9ffb4a8d927545c9a6844946567259a6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9ffb4a8d927545c9a6844946567259a6 2024-11-13T18:37:28,556 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9ffb4a8d927545c9a6844946567259a6, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T18:37:28,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0fb3caed9277459cb165a7e787fd9bb9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0fb3caed9277459cb165a7e787fd9bb9 2024-11-13T18:37:28,579 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0fb3caed9277459cb165a7e787fd9bb9, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T18:37:28,580 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f8326d4028e020fa48ab6da5484ce85e in 559ms, sequenceid=75, compaction requested=false 2024-11-13T18:37:28,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:28,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:28,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T18:37:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T18:37:28,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T18:37:28,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6640 sec 2024-11-13T18:37:28,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.6710 sec 2024-11-13T18:37:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:28,723 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:28,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T18:37:28,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/a21bd90502ea4551aba8748bc4d1556f is 50, key is test_row_0/A:col10/1731523048716/Put/seqid=0 2024-11-13T18:37:28,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523108748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523108749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523108750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523108752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523108750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742042_1218 (size=12001) 2024-11-13T18:37:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523108863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523108864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523108864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523108863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523108864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T18:37:29,063 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:29,064 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-13T18:37:29,066 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:37:29,067 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:29,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:29,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523109073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523109074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523109074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523109075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523109075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:37:29,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/a21bd90502ea4551aba8748bc4d1556f 2024-11-13T18:37:29,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ba40d3b8df604360b5a3eba15d62294f is 50, key is test_row_0/B:col10/1731523048716/Put/seqid=0 2024-11-13T18:37:29,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:37:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:29,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742043_1219 (size=12001) 2024-11-13T18:37:29,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ba40d3b8df604360b5a3eba15d62294f 2024-11-13T18:37:29,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/e8c8360c0fa0476a9ff08d1940a633b9 is 50, key is test_row_0/C:col10/1731523048716/Put/seqid=0 2024-11-13T18:37:29,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742044_1220 (size=12001) 2024-11-13T18:37:29,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:37:29,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:29,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:37:29,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523109383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523109383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523109383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523109383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523109384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:37:29,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:29,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:37:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:29,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/e8c8360c0fa0476a9ff08d1940a633b9 2024-11-13T18:37:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:37:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/a21bd90502ea4551aba8748bc4d1556f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/a21bd90502ea4551aba8748bc4d1556f 2024-11-13T18:37:29,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/a21bd90502ea4551aba8748bc4d1556f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T18:37:29,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ba40d3b8df604360b5a3eba15d62294f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ba40d3b8df604360b5a3eba15d62294f 2024-11-13T18:37:29,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ba40d3b8df604360b5a3eba15d62294f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T18:37:29,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/e8c8360c0fa0476a9ff08d1940a633b9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e8c8360c0fa0476a9ff08d1940a633b9 2024-11-13T18:37:29,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e8c8360c0fa0476a9ff08d1940a633b9, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T18:37:29,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for f8326d4028e020fa48ab6da5484ce85e in 998ms, sequenceid=96, compaction requested=true 2024-11-13T18:37:29,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:29,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:29,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:29,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:29,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:29,721 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:29,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:29,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:37:29,721 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:29,722 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:29,722 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:29,722 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:29,723 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,723 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:29,723 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e3e25d5da6e3437091ca0dd206023b4b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/fb6631fff2024d6ea95bb8bcce78929e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/a21bd90502ea4551aba8748bc4d1556f] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=35.3 K 2024-11-13T18:37:29,723 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,723 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/01fb61d679464ab78a17c6924127139f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9ffb4a8d927545c9a6844946567259a6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ba40d3b8df604360b5a3eba15d62294f] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=35.3 K 2024-11-13T18:37:29,723 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e25d5da6e3437091ca0dd206023b4b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523046197 2024-11-13T18:37:29,723 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6631fff2024d6ea95bb8bcce78929e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731523046876 2024-11-13T18:37:29,724 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01fb61d679464ab78a17c6924127139f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523046197 2024-11-13T18:37:29,725 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ffb4a8d927545c9a6844946567259a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731523046876 2024-11-13T18:37:29,725 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a21bd90502ea4551aba8748bc4d1556f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731523048715 2024-11-13T18:37:29,725 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba40d3b8df604360b5a3eba15d62294f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731523048715 2024-11-13T18:37:29,743 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:29,744 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/c19e6282565a44a8b54198c44818330b is 50, key is test_row_0/A:col10/1731523048716/Put/seqid=0 2024-11-13T18:37:29,751 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:29,752 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/47279ecadc79434982d39454243902c7 is 50, key is test_row_0/B:col10/1731523048716/Put/seqid=0 2024-11-13T18:37:29,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742045_1221 (size=12207) 2024-11-13T18:37:29,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742046_1222 (size=12207) 2024-11-13T18:37:29,788 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/47279ecadc79434982d39454243902c7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/47279ecadc79434982d39454243902c7 2024-11-13T18:37:29,789 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/c19e6282565a44a8b54198c44818330b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c19e6282565a44a8b54198c44818330b 2024-11-13T18:37:29,796 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 47279ecadc79434982d39454243902c7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:29,796 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:29,796 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=13, startTime=1731523049721; duration=0sec 2024-11-13T18:37:29,796 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:29,796 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:29,797 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:29,798 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:29,798 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:29,798 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,798 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into c19e6282565a44a8b54198c44818330b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:29,798 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:29,798 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=13, startTime=1731523049721; duration=0sec 2024-11-13T18:37:29,798 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/c1ba8483b5fa418db0e9b057237700ed, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0fb3caed9277459cb165a7e787fd9bb9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e8c8360c0fa0476a9ff08d1940a633b9] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=35.3 K 2024-11-13T18:37:29,798 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:29,798 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:29,800 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ba8483b5fa418db0e9b057237700ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523046197 2024-11-13T18:37:29,800 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fb3caed9277459cb165a7e787fd9bb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731523046876 2024-11-13T18:37:29,801 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c8360c0fa0476a9ff08d1940a633b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731523048715 2024-11-13T18:37:29,815 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:29,817 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/5f004306f1bd40ba92b6d24b330af1ce is 50, key is test_row_0/C:col10/1731523048716/Put/seqid=0 2024-11-13T18:37:29,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T18:37:29,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:29,838 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T18:37:29,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/54ada0444ff54e3a94963d99bf65376f is 50, key is test_row_0/A:col10/1731523048750/Put/seqid=0 2024-11-13T18:37:29,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742047_1223 (size=12207) 2024-11-13T18:37:29,865 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/5f004306f1bd40ba92b6d24b330af1ce as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5f004306f1bd40ba92b6d24b330af1ce 2024-11-13T18:37:29,876 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into 5f004306f1bd40ba92b6d24b330af1ce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:29,876 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:29,876 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=13, startTime=1731523049721; duration=0sec 2024-11-13T18:37:29,876 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:29,876 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:29,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742048_1224 (size=12001) 2024-11-13T18:37:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:29,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:29,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523109920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523109921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523109922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523109922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523109922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523110033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523110033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523110033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523110033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523110036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:37:30,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523110243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523110243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523110243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523110253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523110253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,282 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/54ada0444ff54e3a94963d99bf65376f 2024-11-13T18:37:30,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/fe37cc97270a4bc68f9c036d412205dd is 50, key is test_row_0/B:col10/1731523048750/Put/seqid=0 2024-11-13T18:37:30,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742049_1225 (size=12001) 2024-11-13T18:37:30,343 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/fe37cc97270a4bc68f9c036d412205dd 2024-11-13T18:37:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/208f78b8bda64d719bc5189a04945766 is 50, key is test_row_0/C:col10/1731523048750/Put/seqid=0 2024-11-13T18:37:30,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742050_1226 (size=12001) 2024-11-13T18:37:30,372 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/208f78b8bda64d719bc5189a04945766 2024-11-13T18:37:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/54ada0444ff54e3a94963d99bf65376f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/54ada0444ff54e3a94963d99bf65376f 2024-11-13T18:37:30,388 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/54ada0444ff54e3a94963d99bf65376f, entries=150, sequenceid=115, filesize=11.7 K 2024-11-13T18:37:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/fe37cc97270a4bc68f9c036d412205dd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/fe37cc97270a4bc68f9c036d412205dd 2024-11-13T18:37:30,399 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/fe37cc97270a4bc68f9c036d412205dd, entries=150, sequenceid=115, filesize=11.7 K 2024-11-13T18:37:30,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/208f78b8bda64d719bc5189a04945766 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/208f78b8bda64d719bc5189a04945766 2024-11-13T18:37:30,408 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/208f78b8bda64d719bc5189a04945766, entries=150, sequenceid=115, filesize=11.7 K 2024-11-13T18:37:30,410 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f8326d4028e020fa48ab6da5484ce85e in 572ms, sequenceid=115, compaction requested=false 2024-11-13T18:37:30,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:30,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:30,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-13T18:37:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-13T18:37:30,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T18:37:30,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-11-13T18:37:30,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.3500 sec 2024-11-13T18:37:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:30,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:30,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T18:37:30,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/485044fc036a488795118a9593d8ed24 is 50, key is test_row_0/A:col10/1731523050557/Put/seqid=0 2024-11-13T18:37:30,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523110579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523110580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523110582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523110580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523110585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742051_1227 (size=14541) 2024-11-13T18:37:30,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523110694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523110694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523110694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523110694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523110703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523110903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523110904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523110904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523110904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:30,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523110917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/485044fc036a488795118a9593d8ed24 2024-11-13T18:37:31,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/5b58b01dbdd64a8999bd44ce6dd3d6b7 is 50, key is test_row_0/B:col10/1731523050557/Put/seqid=0 2024-11-13T18:37:31,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742052_1228 (size=12151) 2024-11-13T18:37:31,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/5b58b01dbdd64a8999bd44ce6dd3d6b7 2024-11-13T18:37:31,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/ae1f5b8f9b3b45779c587a3741fc1e48 is 50, key is test_row_0/C:col10/1731523050557/Put/seqid=0 2024-11-13T18:37:31,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742053_1229 (size=12151) 2024-11-13T18:37:31,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/ae1f5b8f9b3b45779c587a3741fc1e48 2024-11-13T18:37:31,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/485044fc036a488795118a9593d8ed24 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/485044fc036a488795118a9593d8ed24 2024-11-13T18:37:31,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/485044fc036a488795118a9593d8ed24, entries=200, sequenceid=138, filesize=14.2 K 2024-11-13T18:37:31,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/5b58b01dbdd64a8999bd44ce6dd3d6b7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5b58b01dbdd64a8999bd44ce6dd3d6b7 2024-11-13T18:37:31,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5b58b01dbdd64a8999bd44ce6dd3d6b7, entries=150, sequenceid=138, filesize=11.9 K 2024-11-13T18:37:31,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/ae1f5b8f9b3b45779c587a3741fc1e48 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/ae1f5b8f9b3b45779c587a3741fc1e48 2024-11-13T18:37:31,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/ae1f5b8f9b3b45779c587a3741fc1e48, entries=150, sequenceid=138, filesize=11.9 K 2024-11-13T18:37:31,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f8326d4028e020fa48ab6da5484ce85e in 612ms, sequenceid=138, compaction requested=true 2024-11-13T18:37:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:31,169 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:37:31,170 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:31,171 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:31,171 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:31,171 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:31,171 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:31,171 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,171 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,171 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c19e6282565a44a8b54198c44818330b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/54ada0444ff54e3a94963d99bf65376f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/485044fc036a488795118a9593d8ed24] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=37.8 K 2024-11-13T18:37:31,171 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/47279ecadc79434982d39454243902c7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/fe37cc97270a4bc68f9c036d412205dd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5b58b01dbdd64a8999bd44ce6dd3d6b7] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=35.5 K 2024-11-13T18:37:31,172 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 47279ecadc79434982d39454243902c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731523048715 2024-11-13T18:37:31,172 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19e6282565a44a8b54198c44818330b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731523048715 2024-11-13T18:37:31,172 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54ada0444ff54e3a94963d99bf65376f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731523048745 2024-11-13T18:37:31,173 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485044fc036a488795118a9593d8ed24, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523049919 2024-11-13T18:37:31,173 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting fe37cc97270a4bc68f9c036d412205dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731523048745 2024-11-13T18:37:31,174 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b58b01dbdd64a8999bd44ce6dd3d6b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523049919 2024-11-13T18:37:31,191 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:31,191 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/47f7b327bda84314a2ebc7d7e3c7b5e7 is 50, key is test_row_0/A:col10/1731523050557/Put/seqid=0 2024-11-13T18:37:31,208 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:31,208 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/7a55d7c3ad974df9bed37921e103e95a is 50, key is test_row_0/B:col10/1731523050557/Put/seqid=0 2024-11-13T18:37:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T18:37:31,214 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:31,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-13T18:37:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:37:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:31,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T18:37:31,220 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:31,221 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:31,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:31,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742054_1230 (size=12459) 2024-11-13T18:37:31,256 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/47f7b327bda84314a2ebc7d7e3c7b5e7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/47f7b327bda84314a2ebc7d7e3c7b5e7 2024-11-13T18:37:31,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742055_1231 (size=12459) 2024-11-13T18:37:31,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b0d4dae661d74a4e85d41059676a43ec is 50, key is test_row_0/A:col10/1731523051218/Put/seqid=0 2024-11-13T18:37:31,265 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into 47f7b327bda84314a2ebc7d7e3c7b5e7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:31,265 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:31,265 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=13, startTime=1731523051169; duration=0sec 2024-11-13T18:37:31,266 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:31,266 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:31,266 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:31,267 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:31,267 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:31,267 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,267 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5f004306f1bd40ba92b6d24b330af1ce, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/208f78b8bda64d719bc5189a04945766, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/ae1f5b8f9b3b45779c587a3741fc1e48] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=35.5 K 2024-11-13T18:37:31,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523111260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,269 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f004306f1bd40ba92b6d24b330af1ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731523048715 2024-11-13T18:37:31,269 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 208f78b8bda64d719bc5189a04945766, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731523048745 2024-11-13T18:37:31,270 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae1f5b8f9b3b45779c587a3741fc1e48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523049919 2024-11-13T18:37:31,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523111264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523111266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523111267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523111268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742056_1232 (size=14541) 2024-11-13T18:37:31,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b0d4dae661d74a4e85d41059676a43ec 2024-11-13T18:37:31,285 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:31,285 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/7ca4dc7b2ed14009b22f6978d7178830 is 50, key is test_row_0/C:col10/1731523050557/Put/seqid=0 2024-11-13T18:37:31,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/d07ab75fdf10434db5610308a01e8838 is 50, key is test_row_0/B:col10/1731523051218/Put/seqid=0 2024-11-13T18:37:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:37:31,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742057_1233 (size=12459) 2024-11-13T18:37:31,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742058_1234 (size=12151) 2024-11-13T18:37:31,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/d07ab75fdf10434db5610308a01e8838 2024-11-13T18:37:31,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/d62309badfad4c279e31d6791417a18a is 50, key is test_row_0/C:col10/1731523051218/Put/seqid=0 2024-11-13T18:37:31,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:37:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:31,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523111373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742059_1235 (size=12151) 2024-11-13T18:37:31,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/d62309badfad4c279e31d6791417a18a 2024-11-13T18:37:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523111384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523111384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523111384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523111384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b0d4dae661d74a4e85d41059676a43ec as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b0d4dae661d74a4e85d41059676a43ec 2024-11-13T18:37:31,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b0d4dae661d74a4e85d41059676a43ec, entries=200, sequenceid=155, filesize=14.2 K 2024-11-13T18:37:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/d07ab75fdf10434db5610308a01e8838 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/d07ab75fdf10434db5610308a01e8838 2024-11-13T18:37:31,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/d07ab75fdf10434db5610308a01e8838, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T18:37:31,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/d62309badfad4c279e31d6791417a18a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d62309badfad4c279e31d6791417a18a 2024-11-13T18:37:31,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d62309badfad4c279e31d6791417a18a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T18:37:31,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f8326d4028e020fa48ab6da5484ce85e in 203ms, sequenceid=155, compaction requested=false 2024-11-13T18:37:31,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:31,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T18:37:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,529 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T18:37:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:37:31,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/2d5ded9d43b445a99162d0e49498e9dc is 50, key is test_row_0/A:col10/1731523051260/Put/seqid=0 2024-11-13T18:37:31,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742060_1236 (size=12151) 2024-11-13T18:37:31,542 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/2d5ded9d43b445a99162d0e49498e9dc 2024-11-13T18:37:31,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/73fa57761ed24d3292984bb6aefe6961 is 50, key is test_row_0/B:col10/1731523051260/Put/seqid=0 2024-11-13T18:37:31,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742061_1237 (size=12151) 2024-11-13T18:37:31,555 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/73fa57761ed24d3292984bb6aefe6961 2024-11-13T18:37:31,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/8165115bf660443e9e68bd0ef61cb072 is 50, key is test_row_0/C:col10/1731523051260/Put/seqid=0 2024-11-13T18:37:31,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742062_1238 (size=12151) 2024-11-13T18:37:31,571 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/8165115bf660443e9e68bd0ef61cb072 2024-11-13T18:37:31,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/2d5ded9d43b445a99162d0e49498e9dc as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d5ded9d43b445a99162d0e49498e9dc 2024-11-13T18:37:31,582 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d5ded9d43b445a99162d0e49498e9dc, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T18:37:31,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/73fa57761ed24d3292984bb6aefe6961 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/73fa57761ed24d3292984bb6aefe6961 2024-11-13T18:37:31,588 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/73fa57761ed24d3292984bb6aefe6961, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T18:37:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:31,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/8165115bf660443e9e68bd0ef61cb072 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/8165115bf660443e9e68bd0ef61cb072 2024-11-13T18:37:31,597 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/8165115bf660443e9e68bd0ef61cb072, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T18:37:31,598 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=26.84 KB/27480 for f8326d4028e020fa48ab6da5484ce85e in 70ms, sequenceid=175, compaction requested=true 2024-11-13T18:37:31,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:31,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T18:37:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T18:37:31,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T18:37:31,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 378 msec 2024-11-13T18:37:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:31,603 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:31,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:31,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 387 msec 2024-11-13T18:37:31,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/f090432c4a3d470c989237bc3e8b5130 is 50, key is test_row_0/A:col10/1731523051599/Put/seqid=0 2024-11-13T18:37:31,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742063_1239 (size=12151) 2024-11-13T18:37:31,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/f090432c4a3d470c989237bc3e8b5130 2024-11-13T18:37:31,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/e5e631a37360438286d2855cace8b4c9 is 50, key is test_row_0/B:col10/1731523051599/Put/seqid=0 2024-11-13T18:37:31,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742064_1240 (size=12151) 2024-11-13T18:37:31,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/e5e631a37360438286d2855cace8b4c9 2024-11-13T18:37:31,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523111634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523111636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523111637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523111639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523111641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/9893e57b63fd4a25a9216135132e275a is 50, key is test_row_0/C:col10/1731523051599/Put/seqid=0 2024-11-13T18:37:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742065_1241 (size=12151) 2024-11-13T18:37:31,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/9893e57b63fd4a25a9216135132e275a 2024-11-13T18:37:31,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/f090432c4a3d470c989237bc3e8b5130 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f090432c4a3d470c989237bc3e8b5130 2024-11-13T18:37:31,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f090432c4a3d470c989237bc3e8b5130, entries=150, sequenceid=186, filesize=11.9 K 2024-11-13T18:37:31,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/e5e631a37360438286d2855cace8b4c9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e5e631a37360438286d2855cace8b4c9 2024-11-13T18:37:31,665 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/7a55d7c3ad974df9bed37921e103e95a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7a55d7c3ad974df9bed37921e103e95a 2024-11-13T18:37:31,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e5e631a37360438286d2855cace8b4c9, entries=150, sequenceid=186, filesize=11.9 K 2024-11-13T18:37:31,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/9893e57b63fd4a25a9216135132e275a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/9893e57b63fd4a25a9216135132e275a 2024-11-13T18:37:31,671 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 7a55d7c3ad974df9bed37921e103e95a(size=12.2 K), total size for store is 47.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:31,671 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:31,672 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=13, startTime=1731523051169; duration=0sec 2024-11-13T18:37:31,672 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:31,672 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:31,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/9893e57b63fd4a25a9216135132e275a, entries=150, sequenceid=186, filesize=11.9 K 2024-11-13T18:37:31,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f8326d4028e020fa48ab6da5484ce85e in 72ms, sequenceid=186, compaction requested=true 2024-11-13T18:37:31,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:31,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:31,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:31,675 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:31,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:31,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:31,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:31,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:37:31,676 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51302 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:31,676 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:31,676 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,676 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/47f7b327bda84314a2ebc7d7e3c7b5e7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b0d4dae661d74a4e85d41059676a43ec, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d5ded9d43b445a99162d0e49498e9dc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f090432c4a3d470c989237bc3e8b5130] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=50.1 K 2024-11-13T18:37:31,676 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f7b327bda84314a2ebc7d7e3c7b5e7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523049919 2024-11-13T18:37:31,677 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b0d4dae661d74a4e85d41059676a43ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731523050583 2024-11-13T18:37:31,677 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d5ded9d43b445a99162d0e49498e9dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731523051260 2024-11-13T18:37:31,677 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting f090432c4a3d470c989237bc3e8b5130, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731523051599 2024-11-13T18:37:31,687 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:31,688 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/52f84a103a7f4a67849c467c2f0e82ef is 50, key is test_row_0/A:col10/1731523051599/Put/seqid=0 2024-11-13T18:37:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742066_1242 (size=12595) 2024-11-13T18:37:31,749 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/7ca4dc7b2ed14009b22f6978d7178830 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7ca4dc7b2ed14009b22f6978d7178830 2024-11-13T18:37:31,755 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into 7ca4dc7b2ed14009b22f6978d7178830(size=12.2 K), total size for store is 47.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:31,755 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:31,755 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=13, startTime=1731523051169; duration=0sec 2024-11-13T18:37:31,755 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:37:31,755 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:31,755 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:31,755 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:31,758 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:31,758 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:31,758 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:31,758 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7a55d7c3ad974df9bed37921e103e95a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/d07ab75fdf10434db5610308a01e8838, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/73fa57761ed24d3292984bb6aefe6961, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e5e631a37360438286d2855cace8b4c9] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=47.8 K 2024-11-13T18:37:31,758 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a55d7c3ad974df9bed37921e103e95a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523049919 2024-11-13T18:37:31,759 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d07ab75fdf10434db5610308a01e8838, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731523051216 2024-11-13T18:37:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:31,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:31,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:37:31,760 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73fa57761ed24d3292984bb6aefe6961, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731523051260 2024-11-13T18:37:31,761 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5e631a37360438286d2855cace8b4c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731523051599 2024-11-13T18:37:31,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/f6bec67c6a1247a0857119a6ede90300 is 50, key is test_row_0/A:col10/1731523051626/Put/seqid=0 2024-11-13T18:37:31,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742067_1243 (size=16931) 2024-11-13T18:37:31,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/f6bec67c6a1247a0857119a6ede90300 2024-11-13T18:37:31,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523111765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523111765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,795 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:31,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,796 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0912f5fcf75a42698546cb8ed63d9636 is 50, key is test_row_0/B:col10/1731523051599/Put/seqid=0 2024-11-13T18:37:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523111789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523111790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523111796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/274434554b3b4edfa67d9eed6346d60f is 50, key is test_row_0/B:col10/1731523051626/Put/seqid=0 2024-11-13T18:37:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T18:37:31,858 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:31,859 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-13T18:37:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:37:31,862 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:31,863 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:31,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:31,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742069_1245 (size=12151) 2024-11-13T18:37:31,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742068_1244 (size=12595) 2024-11-13T18:37:31,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523111903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523111903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523111904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523111904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523111913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:37:32,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:37:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:32,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:32,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:32,102 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/52f84a103a7f4a67849c467c2f0e82ef as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/52f84a103a7f4a67849c467c2f0e82ef 2024-11-13T18:37:32,108 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into 52f84a103a7f4a67849c467c2f0e82ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:32,108 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:32,108 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=12, startTime=1731523051675; duration=0sec 2024-11-13T18:37:32,108 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:32,108 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:32,108 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:32,110 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:32,110 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:32,110 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:32,110 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7ca4dc7b2ed14009b22f6978d7178830, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d62309badfad4c279e31d6791417a18a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/8165115bf660443e9e68bd0ef61cb072, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/9893e57b63fd4a25a9216135132e275a] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=47.8 K 2024-11-13T18:37:32,112 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca4dc7b2ed14009b22f6978d7178830, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523049919 2024-11-13T18:37:32,113 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting d62309badfad4c279e31d6791417a18a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731523051216 2024-11-13T18:37:32,114 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8165115bf660443e9e68bd0ef61cb072, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731523051260 2024-11-13T18:37:32,114 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9893e57b63fd4a25a9216135132e275a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731523051599 2024-11-13T18:37:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523112113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523112113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523112113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523112114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523112123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,129 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:32,130 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/757356d034744efb9a802372256a1937 is 50, key is test_row_0/C:col10/1731523051599/Put/seqid=0 2024-11-13T18:37:32,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742070_1246 (size=12595) 2024-11-13T18:37:32,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:37:32,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:32,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:32,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:32,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:32,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:37:32,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/274434554b3b4edfa67d9eed6346d60f 2024-11-13T18:37:32,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/7f1ad8fa4bcc40f9951f13472057fdb9 is 50, key is test_row_0/C:col10/1731523051626/Put/seqid=0 2024-11-13T18:37:32,287 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0912f5fcf75a42698546cb8ed63d9636 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0912f5fcf75a42698546cb8ed63d9636 2024-11-13T18:37:32,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742071_1247 (size=12151) 2024-11-13T18:37:32,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/7f1ad8fa4bcc40f9951f13472057fdb9 2024-11-13T18:37:32,294 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 0912f5fcf75a42698546cb8ed63d9636(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:32,294 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:32,294 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=12, startTime=1731523051675; duration=0sec 2024-11-13T18:37:32,294 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:32,294 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:32,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/f6bec67c6a1247a0857119a6ede90300 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f6bec67c6a1247a0857119a6ede90300 2024-11-13T18:37:32,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f6bec67c6a1247a0857119a6ede90300, entries=250, sequenceid=215, filesize=16.5 K 2024-11-13T18:37:32,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/274434554b3b4edfa67d9eed6346d60f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/274434554b3b4edfa67d9eed6346d60f 2024-11-13T18:37:32,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/274434554b3b4edfa67d9eed6346d60f, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T18:37:32,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/7f1ad8fa4bcc40f9951f13472057fdb9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7f1ad8fa4bcc40f9951f13472057fdb9 2024-11-13T18:37:32,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7f1ad8fa4bcc40f9951f13472057fdb9, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T18:37:32,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for f8326d4028e020fa48ab6da5484ce85e in 555ms, sequenceid=215, compaction requested=false 2024-11-13T18:37:32,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:32,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T18:37:32,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:32,325 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-13T18:37:32,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/60e5ebd88dbb4840b9a1723ba09f9aa3 is 50, key is test_row_0/A:col10/1731523051768/Put/seqid=0 2024-11-13T18:37:32,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742072_1248 (size=12151) 2024-11-13T18:37:32,349 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/60e5ebd88dbb4840b9a1723ba09f9aa3 2024-11-13T18:37:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9f446f6018ba41ad960c5292126bffe1 is 50, key is test_row_0/B:col10/1731523051768/Put/seqid=0 2024-11-13T18:37:32,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:32,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742073_1249 (size=12151) 2024-11-13T18:37:32,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523112476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523112479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523112477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523112482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523112481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:37:32,576 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/757356d034744efb9a802372256a1937 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/757356d034744efb9a802372256a1937 2024-11-13T18:37:32,583 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into 757356d034744efb9a802372256a1937(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:32,583 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:32,583 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=12, startTime=1731523051675; duration=0sec 2024-11-13T18:37:32,583 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:32,583 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:32,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523112593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523112594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523112594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523112594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523112594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523112804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523112804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523112804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523112804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523112804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:32,832 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9f446f6018ba41ad960c5292126bffe1 2024-11-13T18:37:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/6c7109e2640e4f4eab276474e4fa0aea is 50, key is test_row_0/C:col10/1731523051768/Put/seqid=0 2024-11-13T18:37:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742074_1250 (size=12151) 2024-11-13T18:37:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:37:33,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523113113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523113113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523113114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523113117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523113117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,256 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/6c7109e2640e4f4eab276474e4fa0aea 2024-11-13T18:37:33,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/60e5ebd88dbb4840b9a1723ba09f9aa3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/60e5ebd88dbb4840b9a1723ba09f9aa3 2024-11-13T18:37:33,334 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/60e5ebd88dbb4840b9a1723ba09f9aa3, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T18:37:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9f446f6018ba41ad960c5292126bffe1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9f446f6018ba41ad960c5292126bffe1 2024-11-13T18:37:33,340 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9f446f6018ba41ad960c5292126bffe1, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T18:37:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/6c7109e2640e4f4eab276474e4fa0aea as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7109e2640e4f4eab276474e4fa0aea 2024-11-13T18:37:33,347 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7109e2640e4f4eab276474e4fa0aea, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T18:37:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-13T18:37:33,359 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for f8326d4028e020fa48ab6da5484ce85e in 1033ms, sequenceid=226, compaction requested=true 2024-11-13T18:37:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-13T18:37:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-13T18:37:33,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-13T18:37:33,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-13T18:37:33,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5040 sec 2024-11-13T18:37:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:33,627 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:33,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-13T18:37:33,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523113631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523113633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523113634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523113634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523113635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/8056ee98b57c406982c52615d836342b is 50, key is test_row_0/A:col10/1731523052477/Put/seqid=0 2024-11-13T18:37:33,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742075_1251 (size=14591) 2024-11-13T18:37:33,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/8056ee98b57c406982c52615d836342b 2024-11-13T18:37:33,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/64d5b3c2ced846d9a4da8b589d251839 is 50, key is test_row_0/B:col10/1731523052477/Put/seqid=0 2024-11-13T18:37:33,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742076_1252 (size=12201) 2024-11-13T18:37:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523113743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523113744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523113744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523113953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523113954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523113954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T18:37:34,013 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:34,014 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-13T18:37:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:37:34,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:34,017 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:34,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:34,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/64d5b3c2ced846d9a4da8b589d251839 2024-11-13T18:37:34,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/e496171c5d1543419661cd0c352aa1e4 is 50, key is test_row_0/C:col10/1731523052477/Put/seqid=0 2024-11-13T18:37:34,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742077_1253 (size=12201) 2024-11-13T18:37:34,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:37:34,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:37:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523114263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523114263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523114263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:37:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:37:34,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:37:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:34,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/e496171c5d1543419661cd0c352aa1e4 2024-11-13T18:37:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/8056ee98b57c406982c52615d836342b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/8056ee98b57c406982c52615d836342b 2024-11-13T18:37:34,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/8056ee98b57c406982c52615d836342b, entries=200, sequenceid=257, filesize=14.2 K 2024-11-13T18:37:34,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/64d5b3c2ced846d9a4da8b589d251839 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/64d5b3c2ced846d9a4da8b589d251839 2024-11-13T18:37:34,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/64d5b3c2ced846d9a4da8b589d251839, entries=150, sequenceid=257, filesize=11.9 K 2024-11-13T18:37:34,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/e496171c5d1543419661cd0c352aa1e4 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e496171c5d1543419661cd0c352aa1e4 2024-11-13T18:37:34,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e496171c5d1543419661cd0c352aa1e4, entries=150, sequenceid=257, filesize=11.9 K 2024-11-13T18:37:34,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for f8326d4028e020fa48ab6da5484ce85e in 910ms, sequenceid=257, compaction requested=true 2024-11-13T18:37:34,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:34,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:34,538 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:34,538 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:34,540 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:34,540 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56268 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:34,540 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:34,540 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:34,540 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,540 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,540 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0912f5fcf75a42698546cb8ed63d9636, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/274434554b3b4edfa67d9eed6346d60f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9f446f6018ba41ad960c5292126bffe1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/64d5b3c2ced846d9a4da8b589d251839] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=47.9 K 2024-11-13T18:37:34,540 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/52f84a103a7f4a67849c467c2f0e82ef, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f6bec67c6a1247a0857119a6ede90300, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/60e5ebd88dbb4840b9a1723ba09f9aa3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/8056ee98b57c406982c52615d836342b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=54.9 K 2024-11-13T18:37:34,541 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0912f5fcf75a42698546cb8ed63d9636, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731523051599 2024-11-13T18:37:34,541 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52f84a103a7f4a67849c467c2f0e82ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731523051599 2024-11-13T18:37:34,541 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6bec67c6a1247a0857119a6ede90300, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731523051626 2024-11-13T18:37:34,541 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 274434554b3b4edfa67d9eed6346d60f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731523051626 2024-11-13T18:37:34,541 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60e5ebd88dbb4840b9a1723ba09f9aa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731523051768 2024-11-13T18:37:34,541 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f446f6018ba41ad960c5292126bffe1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731523051768 2024-11-13T18:37:34,542 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8056ee98b57c406982c52615d836342b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731523052477 2024-11-13T18:37:34,542 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 64d5b3c2ced846d9a4da8b589d251839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731523052477 2024-11-13T18:37:34,554 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:34,554 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/6979842875574e54ba30c2626c58f706 is 50, key is test_row_0/B:col10/1731523052477/Put/seqid=0 2024-11-13T18:37:34,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742078_1254 (size=12781) 2024-11-13T18:37:34,563 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#214 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:34,564 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ba20cf6ec96a4cfb9974773c6000e20c is 50, key is test_row_0/A:col10/1731523052477/Put/seqid=0 2024-11-13T18:37:34,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742079_1255 (size=12781) 2024-11-13T18:37:34,583 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ba20cf6ec96a4cfb9974773c6000e20c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ba20cf6ec96a4cfb9974773c6000e20c 2024-11-13T18:37:34,590 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into ba20cf6ec96a4cfb9974773c6000e20c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:34,590 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:34,590 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=12, startTime=1731523054537; duration=0sec 2024-11-13T18:37:34,590 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:34,591 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:34,591 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:34,592 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:34,592 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:34,592 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,593 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/757356d034744efb9a802372256a1937, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7f1ad8fa4bcc40f9951f13472057fdb9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7109e2640e4f4eab276474e4fa0aea, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e496171c5d1543419661cd0c352aa1e4] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=47.9 K 2024-11-13T18:37:34,593 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 757356d034744efb9a802372256a1937, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731523051599 2024-11-13T18:37:34,593 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f1ad8fa4bcc40f9951f13472057fdb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731523051626 2024-11-13T18:37:34,594 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7109e2640e4f4eab276474e4fa0aea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731523051768 2024-11-13T18:37:34,594 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e496171c5d1543419661cd0c352aa1e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731523052477 2024-11-13T18:37:34,616 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:34,617 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/4aac26c722664d088259115c439b72f4 is 50, key is test_row_0/C:col10/1731523052477/Put/seqid=0 2024-11-13T18:37:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742080_1256 (size=12781) 2024-11-13T18:37:34,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T18:37:34,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:34,632 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-13T18:37:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:37:34,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/86e331c122824f9f89d0ea6dc652c9d6 is 50, key is test_row_1/A:col10/1731523053632/Put/seqid=0 2024-11-13T18:37:34,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742081_1257 (size=9857) 2024-11-13T18:37:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:34,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:34,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523114725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523114729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523114773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523114774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523114774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523114833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523114844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:34,971 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/6979842875574e54ba30c2626c58f706 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/6979842875574e54ba30c2626c58f706 2024-11-13T18:37:34,977 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 6979842875574e54ba30c2626c58f706(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:34,977 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:34,977 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=12, startTime=1731523054538; duration=0sec 2024-11-13T18:37:34,977 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:34,977 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:35,041 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/4aac26c722664d088259115c439b72f4 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4aac26c722664d088259115c439b72f4 2024-11-13T18:37:35,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523115044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,048 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into 4aac26c722664d088259115c439b72f4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:35,048 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:35,048 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=12, startTime=1731523054538; duration=0sec 2024-11-13T18:37:35,048 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:35,048 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:35,054 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/86e331c122824f9f89d0ea6dc652c9d6 2024-11-13T18:37:35,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523115054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9da8a372c2ab48e69eda766164071a8f is 50, key is test_row_1/B:col10/1731523053632/Put/seqid=0 2024-11-13T18:37:35,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742082_1258 (size=9857) 2024-11-13T18:37:35,098 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9da8a372c2ab48e69eda766164071a8f 2024-11-13T18:37:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/fd4d493990504072ba3e248dbe530c93 is 50, key is test_row_1/C:col10/1731523053632/Put/seqid=0 2024-11-13T18:37:35,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742083_1259 (size=9857) 2024-11-13T18:37:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:37:35,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523115353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523115364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,548 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/fd4d493990504072ba3e248dbe530c93 2024-11-13T18:37:35,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/86e331c122824f9f89d0ea6dc652c9d6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/86e331c122824f9f89d0ea6dc652c9d6 2024-11-13T18:37:35,559 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/86e331c122824f9f89d0ea6dc652c9d6, entries=100, sequenceid=264, filesize=9.6 K 2024-11-13T18:37:35,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/9da8a372c2ab48e69eda766164071a8f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9da8a372c2ab48e69eda766164071a8f 2024-11-13T18:37:35,566 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9da8a372c2ab48e69eda766164071a8f, entries=100, sequenceid=264, filesize=9.6 K 2024-11-13T18:37:35,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/fd4d493990504072ba3e248dbe530c93 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fd4d493990504072ba3e248dbe530c93 2024-11-13T18:37:35,573 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fd4d493990504072ba3e248dbe530c93, entries=100, sequenceid=264, filesize=9.6 K 2024-11-13T18:37:35,574 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for f8326d4028e020fa48ab6da5484ce85e in 942ms, sequenceid=264, compaction requested=false 2024-11-13T18:37:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-13T18:37:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-13T18:37:35,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T18:37:35,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5590 sec 2024-11-13T18:37:35,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5710 sec 2024-11-13T18:37:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:35,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-13T18:37:35,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523115787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523115788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523115789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/7cb733d7a80046e395b34db3f7d6bd5e is 50, key is test_row_0/A:col10/1731523055786/Put/seqid=0 2024-11-13T18:37:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742084_1260 (size=17181) 2024-11-13T18:37:35,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/7cb733d7a80046e395b34db3f7d6bd5e 2024-11-13T18:37:35,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/19d813468ded4daaa3aa677adeee38ff is 50, key is test_row_0/B:col10/1731523055786/Put/seqid=0 2024-11-13T18:37:35,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742085_1261 (size=12301) 2024-11-13T18:37:35,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523115863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523115873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523115903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523115903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523115903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:35,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:35,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523116113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523116113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523116113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T18:37:36,164 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:36,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T18:37:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:37:36,166 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:36,167 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:36,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:36,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/19d813468ded4daaa3aa677adeee38ff 2024-11-13T18:37:36,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/6e5128428d484b969297bd1e8919eab6 is 50, key is test_row_0/C:col10/1731523055786/Put/seqid=0 2024-11-13T18:37:36,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742086_1262 (size=12301) 2024-11-13T18:37:36,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/6e5128428d484b969297bd1e8919eab6 2024-11-13T18:37:36,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/7cb733d7a80046e395b34db3f7d6bd5e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/7cb733d7a80046e395b34db3f7d6bd5e 2024-11-13T18:37:36,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/7cb733d7a80046e395b34db3f7d6bd5e, entries=250, sequenceid=299, filesize=16.8 K 2024-11-13T18:37:36,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/19d813468ded4daaa3aa677adeee38ff as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/19d813468ded4daaa3aa677adeee38ff 2024-11-13T18:37:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:37:36,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/19d813468ded4daaa3aa677adeee38ff, entries=150, sequenceid=299, filesize=12.0 K 2024-11-13T18:37:36,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/6e5128428d484b969297bd1e8919eab6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6e5128428d484b969297bd1e8919eab6 2024-11-13T18:37:36,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6e5128428d484b969297bd1e8919eab6, entries=150, sequenceid=299, filesize=12.0 K 2024-11-13T18:37:36,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for f8326d4028e020fa48ab6da5484ce85e in 498ms, sequenceid=299, compaction requested=true 2024-11-13T18:37:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:36,285 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:36,285 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:36,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:36,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:36,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:36,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:36,288 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39819 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:36,288 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:36,288 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,288 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ba20cf6ec96a4cfb9974773c6000e20c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/86e331c122824f9f89d0ea6dc652c9d6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/7cb733d7a80046e395b34db3f7d6bd5e] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=38.9 K 2024-11-13T18:37:36,289 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:36,289 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:36,289 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,289 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/6979842875574e54ba30c2626c58f706, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9da8a372c2ab48e69eda766164071a8f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/19d813468ded4daaa3aa677adeee38ff] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=34.1 K 2024-11-13T18:37:36,289 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba20cf6ec96a4cfb9974773c6000e20c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731523052477 2024-11-13T18:37:36,290 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6979842875574e54ba30c2626c58f706, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731523052477 2024-11-13T18:37:36,291 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86e331c122824f9f89d0ea6dc652c9d6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731523053631 2024-11-13T18:37:36,291 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da8a372c2ab48e69eda766164071a8f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731523053631 2024-11-13T18:37:36,292 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d813468ded4daaa3aa677adeee38ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731523054727 2024-11-13T18:37:36,292 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cb733d7a80046e395b34db3f7d6bd5e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731523054714 2024-11-13T18:37:36,316 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:36,317 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/5783c12a48f640ea9cd853c87cca6d54 is 50, key is test_row_0/B:col10/1731523055786/Put/seqid=0 2024-11-13T18:37:36,319 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:36,320 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/e928cba5fce24ff4b6c6ca9cfe90c790 is 50, key is test_row_0/A:col10/1731523055786/Put/seqid=0 2024-11-13T18:37:36,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T18:37:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T18:37:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T18:37:36,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T18:37:36,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-13T18:37:36,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 163 msec 2024-11-13T18:37:36,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742087_1263 (size=12983) 2024-11-13T18:37:36,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742088_1264 (size=12983) 2024-11-13T18:37:36,356 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/e928cba5fce24ff4b6c6ca9cfe90c790 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e928cba5fce24ff4b6c6ca9cfe90c790 2024-11-13T18:37:36,364 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into e928cba5fce24ff4b6c6ca9cfe90c790(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:36,364 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:36,364 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=13, startTime=1731523056285; duration=0sec 2024-11-13T18:37:36,365 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:36,365 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:36,365 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:36,366 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:36,366 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:36,366 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,366 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4aac26c722664d088259115c439b72f4, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fd4d493990504072ba3e248dbe530c93, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6e5128428d484b969297bd1e8919eab6] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=34.1 K 2024-11-13T18:37:36,367 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aac26c722664d088259115c439b72f4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731523052477 2024-11-13T18:37:36,367 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd4d493990504072ba3e248dbe530c93, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731523053631 2024-11-13T18:37:36,368 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e5128428d484b969297bd1e8919eab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731523054727 2024-11-13T18:37:36,378 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:36,379 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0a2a4966ef164ed483bf2659b13ee60f is 50, key is test_row_0/C:col10/1731523055786/Put/seqid=0 2024-11-13T18:37:36,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742089_1265 (size=12983) 2024-11-13T18:37:36,393 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0a2a4966ef164ed483bf2659b13ee60f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a2a4966ef164ed483bf2659b13ee60f 2024-11-13T18:37:36,400 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into 0a2a4966ef164ed483bf2659b13ee60f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:36,400 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:36,400 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=13, startTime=1731523056286; duration=0sec 2024-11-13T18:37:36,400 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:36,400 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:36,434 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:36,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:36,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/408d9d03aaa44ff2b96e3581a91dca5c is 50, key is test_row_0/A:col10/1731523056429/Put/seqid=0 2024-11-13T18:37:36,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742090_1266 (size=12301) 2024-11-13T18:37:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T18:37:36,484 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:36,484 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523116479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T18:37:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:37:36,487 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:36,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523116480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523116481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,488 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:36,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:37:36,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523116593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523116593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523116593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:37:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,748 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/5783c12a48f640ea9cd853c87cca6d54 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5783c12a48f640ea9cd853c87cca6d54 2024-11-13T18:37:36,754 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 5783c12a48f640ea9cd853c87cca6d54(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:36,754 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:36,754 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=13, startTime=1731523056285; duration=0sec 2024-11-13T18:37:36,755 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:36,755 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:36,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:37:36,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:36,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:37:36,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523116804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523116804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523116805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/408d9d03aaa44ff2b96e3581a91dca5c 2024-11-13T18:37:36,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/1ced747f894441bebfebfeece83b0ef6 is 50, key is test_row_0/B:col10/1731523056429/Put/seqid=0 2024-11-13T18:37:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523116883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523116883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:36,887 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in f8326d4028e020fa48ab6da5484ce85e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,888 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in f8326d4028e020fa48ab6da5484ce85e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:36,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742091_1267 (size=12301) 2024-11-13T18:37:36,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/1ced747f894441bebfebfeece83b0ef6 2024-11-13T18:37:36,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:37:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:36,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/4765b6363fa346789d6ed53ac6611b22 is 50, key is test_row_0/C:col10/1731523056429/Put/seqid=0 2024-11-13T18:37:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:36,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742092_1268 (size=12301) 2024-11-13T18:37:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:37:37,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:37:37,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:37,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:37,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:37,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523117113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523117113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523117113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:37:37,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:37,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:37,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/4765b6363fa346789d6ed53ac6611b22 2024-11-13T18:37:37,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/408d9d03aaa44ff2b96e3581a91dca5c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/408d9d03aaa44ff2b96e3581a91dca5c 2024-11-13T18:37:37,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:37:37,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:37,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:37,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:37,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:37,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/408d9d03aaa44ff2b96e3581a91dca5c, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T18:37:37,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/1ced747f894441bebfebfeece83b0ef6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/1ced747f894441bebfebfeece83b0ef6 2024-11-13T18:37:37,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/1ced747f894441bebfebfeece83b0ef6, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T18:37:37,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/4765b6363fa346789d6ed53ac6611b22 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4765b6363fa346789d6ed53ac6611b22 2024-11-13T18:37:37,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4765b6363fa346789d6ed53ac6611b22, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T18:37:37,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for f8326d4028e020fa48ab6da5484ce85e in 1067ms, sequenceid=313, compaction requested=false 2024-11-13T18:37:37,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:37,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T18:37:37,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:37,583 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:37:37,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b14747f41d3a476186ad0b940b92927a is 50, key is test_row_0/A:col10/1731523056477/Put/seqid=0 2024-11-13T18:37:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:37:37,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:37,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742093_1269 (size=12301) 2024-11-13T18:37:37,649 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b14747f41d3a476186ad0b940b92927a 2024-11-13T18:37:37,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523117647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523117648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523117650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ea0aff93e94e4104ae30571202480187 is 50, key is test_row_0/B:col10/1731523056477/Put/seqid=0 2024-11-13T18:37:37,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742094_1270 (size=12301) 2024-11-13T18:37:37,761 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ea0aff93e94e4104ae30571202480187 2024-11-13T18:37:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/2b102683097b42beb549c64b036b9d17 is 50, key is test_row_0/C:col10/1731523056477/Put/seqid=0 2024-11-13T18:37:37,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523117764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523117765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523117764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742095_1271 (size=12301) 2024-11-13T18:37:37,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523117983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523117984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:37,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523117993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,224 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/2b102683097b42beb549c64b036b9d17 2024-11-13T18:37:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b14747f41d3a476186ad0b940b92927a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b14747f41d3a476186ad0b940b92927a 2024-11-13T18:37:38,237 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b14747f41d3a476186ad0b940b92927a, entries=150, sequenceid=340, filesize=12.0 K 2024-11-13T18:37:38,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ea0aff93e94e4104ae30571202480187 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ea0aff93e94e4104ae30571202480187 2024-11-13T18:37:38,246 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ea0aff93e94e4104ae30571202480187, entries=150, sequenceid=340, filesize=12.0 K 2024-11-13T18:37:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/2b102683097b42beb549c64b036b9d17 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/2b102683097b42beb549c64b036b9d17 2024-11-13T18:37:38,255 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/2b102683097b42beb549c64b036b9d17, entries=150, sequenceid=340, filesize=12.0 K 2024-11-13T18:37:38,257 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f8326d4028e020fa48ab6da5484ce85e in 674ms, sequenceid=340, compaction requested=true 2024-11-13T18:37:38,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T18:37:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T18:37:38,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T18:37:38,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7720 sec 2024-11-13T18:37:38,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7800 sec 2024-11-13T18:37:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:38,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:38,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b45a2cc79fc4430a8303244be81f91aa is 50, key is test_row_0/A:col10/1731523057642/Put/seqid=0 2024-11-13T18:37:38,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742096_1272 (size=17181) 2024-11-13T18:37:38,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b45a2cc79fc4430a8303244be81f91aa 2024-11-13T18:37:38,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0a16d2541427463c819a4e44361e1a61 is 50, key is test_row_0/B:col10/1731523057642/Put/seqid=0 2024-11-13T18:37:38,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523118387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523118389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523118402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742097_1273 (size=12301) 2024-11-13T18:37:38,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0a16d2541427463c819a4e44361e1a61 2024-11-13T18:37:38,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/5d5de00f2a474a76b6c1a8c260b01470 is 50, key is test_row_0/C:col10/1731523057642/Put/seqid=0 2024-11-13T18:37:38,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742098_1274 (size=12301) 2024-11-13T18:37:38,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/5d5de00f2a474a76b6c1a8c260b01470 2024-11-13T18:37:38,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b45a2cc79fc4430a8303244be81f91aa as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b45a2cc79fc4430a8303244be81f91aa 2024-11-13T18:37:38,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b45a2cc79fc4430a8303244be81f91aa, entries=250, sequenceid=352, filesize=16.8 K 2024-11-13T18:37:38,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0a16d2541427463c819a4e44361e1a61 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0a16d2541427463c819a4e44361e1a61 2024-11-13T18:37:38,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0a16d2541427463c819a4e44361e1a61, entries=150, sequenceid=352, filesize=12.0 K 2024-11-13T18:37:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/5d5de00f2a474a76b6c1a8c260b01470 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5d5de00f2a474a76b6c1a8c260b01470 2024-11-13T18:37:38,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5d5de00f2a474a76b6c1a8c260b01470, entries=150, sequenceid=352, filesize=12.0 K 2024-11-13T18:37:38,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f8326d4028e020fa48ab6da5484ce85e in 200ms, sequenceid=352, compaction requested=true 2024-11-13T18:37:38,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:38,498 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:38,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:38,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:38,499 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:38,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:38,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:38,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:38,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:38,501 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:38,501 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:38,501 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:38,501 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5783c12a48f640ea9cd853c87cca6d54, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/1ced747f894441bebfebfeece83b0ef6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ea0aff93e94e4104ae30571202480187, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0a16d2541427463c819a4e44361e1a61] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=48.7 K 2024-11-13T18:37:38,501 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:38,502 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:38,502 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:38,502 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e928cba5fce24ff4b6c6ca9cfe90c790, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/408d9d03aaa44ff2b96e3581a91dca5c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b14747f41d3a476186ad0b940b92927a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b45a2cc79fc4430a8303244be81f91aa] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=53.5 K 2024-11-13T18:37:38,502 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5783c12a48f640ea9cd853c87cca6d54, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731523054727 2024-11-13T18:37:38,502 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e928cba5fce24ff4b6c6ca9cfe90c790, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731523054727 2024-11-13T18:37:38,505 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ced747f894441bebfebfeece83b0ef6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731523056429 2024-11-13T18:37:38,506 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408d9d03aaa44ff2b96e3581a91dca5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731523056429 2024-11-13T18:37:38,506 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0aff93e94e4104ae30571202480187, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731523056457 2024-11-13T18:37:38,506 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b14747f41d3a476186ad0b940b92927a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731523056457 2024-11-13T18:37:38,506 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a16d2541427463c819a4e44361e1a61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731523057642 2024-11-13T18:37:38,507 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b45a2cc79fc4430a8303244be81f91aa, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731523057641 2024-11-13T18:37:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:38,510 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:38,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:37:38,524 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:38,525 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/b49cfa6615b84a439629f96de118e7ee is 50, key is test_row_0/B:col10/1731523057642/Put/seqid=0 2024-11-13T18:37:38,533 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:38,534 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/96f4dd116eec4e85bdf1f043e911da19 is 50, key is test_row_0/A:col10/1731523057642/Put/seqid=0 2024-11-13T18:37:38,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523118538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523118541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523118542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742099_1275 (size=13119) 2024-11-13T18:37:38,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/840b4da3840647b5b7d742aa49e1e0a9 is 50, key is test_row_0/A:col10/1731523058393/Put/seqid=0 2024-11-13T18:37:38,569 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/b49cfa6615b84a439629f96de118e7ee as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/b49cfa6615b84a439629f96de118e7ee 2024-11-13T18:37:38,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742100_1276 (size=13119) 2024-11-13T18:37:38,575 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into b49cfa6615b84a439629f96de118e7ee(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:38,575 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:38,575 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=12, startTime=1731523058499; duration=0sec 2024-11-13T18:37:38,575 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:38,575 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:38,575 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:37:38,576 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:37:38,577 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:38,577 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:38,586 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a2a4966ef164ed483bf2659b13ee60f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4765b6363fa346789d6ed53ac6611b22, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/2b102683097b42beb549c64b036b9d17, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5d5de00f2a474a76b6c1a8c260b01470] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=48.7 K 2024-11-13T18:37:38,589 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a2a4966ef164ed483bf2659b13ee60f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731523054727 2024-11-13T18:37:38,590 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4765b6363fa346789d6ed53ac6611b22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731523056429 2024-11-13T18:37:38,591 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b102683097b42beb549c64b036b9d17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731523056457 2024-11-13T18:37:38,591 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/96f4dd116eec4e85bdf1f043e911da19 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/96f4dd116eec4e85bdf1f043e911da19 2024-11-13T18:37:38,593 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d5de00f2a474a76b6c1a8c260b01470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731523057642 2024-11-13T18:37:38,599 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into 96f4dd116eec4e85bdf1f043e911da19(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:38,599 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:38,599 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=12, startTime=1731523058498; duration=0sec 2024-11-13T18:37:38,599 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:38,599 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:38,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742101_1277 (size=14741) 2024-11-13T18:37:38,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/840b4da3840647b5b7d742aa49e1e0a9 2024-11-13T18:37:38,624 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:38,625 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/da7a4cfeff9b433e970335c3c145aaf8 is 50, key is test_row_0/C:col10/1731523057642/Put/seqid=0 2024-11-13T18:37:38,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/67fa9b09a94d4527bcbca42828a2ab34 is 50, key is test_row_0/B:col10/1731523058393/Put/seqid=0 2024-11-13T18:37:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T18:37:38,644 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:38,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T18:37:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:37:38,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:38,648 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:38,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523118654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523118654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523118654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742102_1278 (size=13119) 2024-11-13T18:37:38,680 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/da7a4cfeff9b433e970335c3c145aaf8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/da7a4cfeff9b433e970335c3c145aaf8 2024-11-13T18:37:38,686 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into da7a4cfeff9b433e970335c3c145aaf8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:38,686 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:38,686 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=12, startTime=1731523058500; duration=0sec 2024-11-13T18:37:38,687 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:38,687 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:38,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742103_1279 (size=12301) 2024-11-13T18:37:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:37:38,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:38,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:38,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:38,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:38,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:38,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523118864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523118864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523118864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523118893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,896 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in f8326d4028e020fa48ab6da5484ce85e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523118904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:38,906 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in f8326d4028e020fa48ab6da5484ce85e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:38,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:38,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:38,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:38,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:38,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:38,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:38,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:38,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:37:39,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/67fa9b09a94d4527bcbca42828a2ab34 2024-11-13T18:37:39,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:39,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:39,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0a35dc536d4b4a8eb7d683a3dce53579 is 50, key is test_row_0/C:col10/1731523058393/Put/seqid=0 2024-11-13T18:37:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742104_1280 (size=12301) 2024-11-13T18:37:39,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523119173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523119173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:39,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523119173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:39,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:39,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:39,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:37:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:39,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:39,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0a35dc536d4b4a8eb7d683a3dce53579 2024-11-13T18:37:39,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:39,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:39,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/840b4da3840647b5b7d742aa49e1e0a9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/840b4da3840647b5b7d742aa49e1e0a9 2024-11-13T18:37:39,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/840b4da3840647b5b7d742aa49e1e0a9, entries=200, sequenceid=378, filesize=14.4 K 2024-11-13T18:37:39,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/67fa9b09a94d4527bcbca42828a2ab34 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/67fa9b09a94d4527bcbca42828a2ab34 2024-11-13T18:37:39,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/67fa9b09a94d4527bcbca42828a2ab34, entries=150, sequenceid=378, filesize=12.0 K 2024-11-13T18:37:39,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0a35dc536d4b4a8eb7d683a3dce53579 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a35dc536d4b4a8eb7d683a3dce53579 2024-11-13T18:37:39,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a35dc536d4b4a8eb7d683a3dce53579, entries=150, sequenceid=378, filesize=12.0 K 2024-11-13T18:37:39,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f8326d4028e020fa48ab6da5484ce85e in 1099ms, sequenceid=378, compaction requested=false 2024-11-13T18:37:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:39,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:37:39,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/0eeebbd32f9b408bbd6ccd4db949884a is 50, key is test_row_0/A:col10/1731523059686/Put/seqid=0 2024-11-13T18:37:39,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742105_1281 (size=12301) 2024-11-13T18:37:39,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/0eeebbd32f9b408bbd6ccd4db949884a 2024-11-13T18:37:39,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/93f8c6054408431e94c80ae835a2015c is 50, key is test_row_0/B:col10/1731523059686/Put/seqid=0 2024-11-13T18:37:39,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:39,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:39,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742106_1282 (size=12301) 2024-11-13T18:37:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:37:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523119788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523119788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523119789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:39,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:39,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:39,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523119903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523119903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523119903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:39,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:40,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:40,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523120113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523120114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523120114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/93f8c6054408431e94c80ae835a2015c 2024-11-13T18:37:40,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/3b063fc89f3e40ba91fbae4926120b6e is 50, key is test_row_0/C:col10/1731523059686/Put/seqid=0 2024-11-13T18:37:40,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742107_1283 (size=12301) 2024-11-13T18:37:40,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:40,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:40,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523120423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523120424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523120424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:40,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:40,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:40,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/3b063fc89f3e40ba91fbae4926120b6e 2024-11-13T18:37:40,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/0eeebbd32f9b408bbd6ccd4db949884a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/0eeebbd32f9b408bbd6ccd4db949884a 2024-11-13T18:37:40,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/0eeebbd32f9b408bbd6ccd4db949884a, entries=150, sequenceid=393, filesize=12.0 K 2024-11-13T18:37:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/93f8c6054408431e94c80ae835a2015c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/93f8c6054408431e94c80ae835a2015c 2024-11-13T18:37:40,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/93f8c6054408431e94c80ae835a2015c, entries=150, sequenceid=393, filesize=12.0 K 2024-11-13T18:37:40,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/3b063fc89f3e40ba91fbae4926120b6e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/3b063fc89f3e40ba91fbae4926120b6e 2024-11-13T18:37:40,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/3b063fc89f3e40ba91fbae4926120b6e, entries=150, sequenceid=393, filesize=12.0 K 2024-11-13T18:37:40,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for f8326d4028e020fa48ab6da5484ce85e in 925ms, sequenceid=393, compaction requested=true 2024-11-13T18:37:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:40,613 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:40,613 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:40,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:40,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:40,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:40,615 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:40,615 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:40,615 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,615 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/96f4dd116eec4e85bdf1f043e911da19, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/840b4da3840647b5b7d742aa49e1e0a9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/0eeebbd32f9b408bbd6ccd4db949884a] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=39.2 K 2024-11-13T18:37:40,616 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:40,616 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f4dd116eec4e85bdf1f043e911da19, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731523057642 2024-11-13T18:37:40,616 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:40,616 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,616 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/b49cfa6615b84a439629f96de118e7ee, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/67fa9b09a94d4527bcbca42828a2ab34, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/93f8c6054408431e94c80ae835a2015c] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=36.8 K 2024-11-13T18:37:40,616 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b49cfa6615b84a439629f96de118e7ee, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731523057642 2024-11-13T18:37:40,616 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 840b4da3840647b5b7d742aa49e1e0a9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731523058382 2024-11-13T18:37:40,617 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eeebbd32f9b408bbd6ccd4db949884a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731523058539 2024-11-13T18:37:40,617 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 67fa9b09a94d4527bcbca42828a2ab34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731523058393 2024-11-13T18:37:40,618 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f8c6054408431e94c80ae835a2015c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731523058539 2024-11-13T18:37:40,635 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:40,635 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/4cbf7b058c874744a0a60e5f2c1cfc4e is 50, key is test_row_0/A:col10/1731523059686/Put/seqid=0 2024-11-13T18:37:40,638 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:40,639 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0c8784ad59024191bdbf00bc43578216 is 50, key is test_row_0/B:col10/1731523059686/Put/seqid=0 2024-11-13T18:37:40,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T18:37:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,662 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:37:40,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742108_1284 (size=13221) 2024-11-13T18:37:40,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/6a5ea0e0020c4590a0017c532ce2f6c6 is 50, key is test_row_0/A:col10/1731523059770/Put/seqid=0 2024-11-13T18:37:40,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742109_1285 (size=13221) 2024-11-13T18:37:40,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742110_1286 (size=12301) 2024-11-13T18:37:40,719 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/6a5ea0e0020c4590a0017c532ce2f6c6 2024-11-13T18:37:40,722 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0c8784ad59024191bdbf00bc43578216 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0c8784ad59024191bdbf00bc43578216 2024-11-13T18:37:40,728 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 0c8784ad59024191bdbf00bc43578216(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:40,728 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:40,728 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=13, startTime=1731523060613; duration=0sec 2024-11-13T18:37:40,728 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:40,728 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:40,728 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:40,729 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:40,729 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:40,730 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:40,730 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/da7a4cfeff9b433e970335c3c145aaf8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a35dc536d4b4a8eb7d683a3dce53579, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/3b063fc89f3e40ba91fbae4926120b6e] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=36.8 K 2024-11-13T18:37:40,730 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting da7a4cfeff9b433e970335c3c145aaf8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731523057642 2024-11-13T18:37:40,732 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a35dc536d4b4a8eb7d683a3dce53579, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731523058393 2024-11-13T18:37:40,732 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b063fc89f3e40ba91fbae4926120b6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731523058539 2024-11-13T18:37:40,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/c1cea3ffd32940119b055ad645f9890c is 50, key is test_row_0/B:col10/1731523059770/Put/seqid=0 2024-11-13T18:37:40,740 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:40,740 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/a675eace4a834a64bf35b05a7960fbf9 is 50, key is test_row_0/C:col10/1731523059686/Put/seqid=0 2024-11-13T18:37:40,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742112_1288 (size=13221) 2024-11-13T18:37:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:37:40,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742111_1287 (size=12301) 2024-11-13T18:37:40,796 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/c1cea3ffd32940119b055ad645f9890c 2024-11-13T18:37:40,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/fca36b54dc7f43a4b90327c5cd8a1b28 is 50, key is test_row_0/C:col10/1731523059770/Put/seqid=0 2024-11-13T18:37:40,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742113_1289 (size=12301) 2024-11-13T18:37:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:40,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523120949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523120949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523120950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523121070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523121071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523121071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,107 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/4cbf7b058c874744a0a60e5f2c1cfc4e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/4cbf7b058c874744a0a60e5f2c1cfc4e 2024-11-13T18:37:41,129 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into 4cbf7b058c874744a0a60e5f2c1cfc4e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:41,129 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:41,130 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=13, startTime=1731523060613; duration=0sec 2024-11-13T18:37:41,130 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:41,130 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:41,188 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/a675eace4a834a64bf35b05a7960fbf9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/a675eace4a834a64bf35b05a7960fbf9 2024-11-13T18:37:41,194 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into a675eace4a834a64bf35b05a7960fbf9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:41,194 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:41,194 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=13, startTime=1731523060614; duration=0sec 2024-11-13T18:37:41,194 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:41,194 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:41,251 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/fca36b54dc7f43a4b90327c5cd8a1b28 2024-11-13T18:37:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/6a5ea0e0020c4590a0017c532ce2f6c6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6a5ea0e0020c4590a0017c532ce2f6c6 2024-11-13T18:37:41,260 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6a5ea0e0020c4590a0017c532ce2f6c6, entries=150, sequenceid=419, filesize=12.0 K 2024-11-13T18:37:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/c1cea3ffd32940119b055ad645f9890c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c1cea3ffd32940119b055ad645f9890c 2024-11-13T18:37:41,265 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c1cea3ffd32940119b055ad645f9890c, entries=150, sequenceid=419, filesize=12.0 K 2024-11-13T18:37:41,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/fca36b54dc7f43a4b90327c5cd8a1b28 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fca36b54dc7f43a4b90327c5cd8a1b28 2024-11-13T18:37:41,270 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fca36b54dc7f43a4b90327c5cd8a1b28, entries=150, sequenceid=419, filesize=12.0 K 2024-11-13T18:37:41,271 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f8326d4028e020fa48ab6da5484ce85e in 609ms, sequenceid=419, compaction requested=false 2024-11-13T18:37:41,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:41,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:41,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T18:37:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T18:37:41,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T18:37:41,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6250 sec 2024-11-13T18:37:41,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.6330 sec 2024-11-13T18:37:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:41,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:41,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:41,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/c0908c00453040e1afe7eea4a288b25b is 50, key is test_row_0/A:col10/1731523060948/Put/seqid=0 2024-11-13T18:37:41,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742114_1290 (size=14741) 2024-11-13T18:37:41,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/c0908c00453040e1afe7eea4a288b25b 2024-11-13T18:37:41,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ee59adfe525946e28716d2ba6a4d3e1e is 50, key is test_row_0/B:col10/1731523060948/Put/seqid=0 2024-11-13T18:37:41,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523121348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742115_1291 (size=12301) 2024-11-13T18:37:41,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523121349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523121351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523121464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523121464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523121464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523121673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523121674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523121674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:41,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ee59adfe525946e28716d2ba6a4d3e1e 2024-11-13T18:37:41,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/af5a4c8135f84a32b43067b05f11cfaa is 50, key is test_row_0/C:col10/1731523060948/Put/seqid=0 2024-11-13T18:37:41,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742116_1292 (size=12301) 2024-11-13T18:37:41,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/af5a4c8135f84a32b43067b05f11cfaa 2024-11-13T18:37:41,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/c0908c00453040e1afe7eea4a288b25b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c0908c00453040e1afe7eea4a288b25b 2024-11-13T18:37:41,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c0908c00453040e1afe7eea4a288b25b, entries=200, sequenceid=435, filesize=14.4 K 2024-11-13T18:37:41,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/ee59adfe525946e28716d2ba6a4d3e1e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ee59adfe525946e28716d2ba6a4d3e1e 2024-11-13T18:37:41,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ee59adfe525946e28716d2ba6a4d3e1e, entries=150, sequenceid=435, filesize=12.0 K 2024-11-13T18:37:41,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/af5a4c8135f84a32b43067b05f11cfaa as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/af5a4c8135f84a32b43067b05f11cfaa 2024-11-13T18:37:41,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/af5a4c8135f84a32b43067b05f11cfaa, entries=150, sequenceid=435, filesize=12.0 K 2024-11-13T18:37:41,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f8326d4028e020fa48ab6da5484ce85e in 522ms, sequenceid=435, compaction requested=true 2024-11-13T18:37:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:41,810 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:41,810 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:41,811 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:41,811 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:41,811 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:41,811 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/4cbf7b058c874744a0a60e5f2c1cfc4e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6a5ea0e0020c4590a0017c532ce2f6c6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c0908c00453040e1afe7eea4a288b25b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=39.3 K 2024-11-13T18:37:41,811 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cbf7b058c874744a0a60e5f2c1cfc4e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731523058539 2024-11-13T18:37:41,811 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:41,812 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:41,812 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:41,812 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0c8784ad59024191bdbf00bc43578216, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c1cea3ffd32940119b055ad645f9890c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ee59adfe525946e28716d2ba6a4d3e1e] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=36.9 K 2024-11-13T18:37:41,812 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5ea0e0020c4590a0017c532ce2f6c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731523059770 2024-11-13T18:37:41,812 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0908c00453040e1afe7eea4a288b25b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731523060948 2024-11-13T18:37:41,813 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c8784ad59024191bdbf00bc43578216, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731523058539 2024-11-13T18:37:41,813 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c1cea3ffd32940119b055ad645f9890c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731523059770 2024-11-13T18:37:41,814 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ee59adfe525946e28716d2ba6a4d3e1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731523060948 2024-11-13T18:37:41,831 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:41,832 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/c20e5ed0d34e43379eadb267178146f3 is 50, key is test_row_0/A:col10/1731523060948/Put/seqid=0 2024-11-13T18:37:41,839 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:41,839 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/3316a37cde034a00aea09dcc2ac7ec03 is 50, key is test_row_0/B:col10/1731523060948/Put/seqid=0 2024-11-13T18:37:41,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742117_1293 (size=13323) 2024-11-13T18:37:41,849 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/c20e5ed0d34e43379eadb267178146f3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c20e5ed0d34e43379eadb267178146f3 2024-11-13T18:37:41,854 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into c20e5ed0d34e43379eadb267178146f3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:41,854 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:41,854 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=13, startTime=1731523061810; duration=0sec 2024-11-13T18:37:41,855 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:41,855 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:41,855 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:41,856 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:41,856 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:41,856 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:41,856 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/a675eace4a834a64bf35b05a7960fbf9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fca36b54dc7f43a4b90327c5cd8a1b28, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/af5a4c8135f84a32b43067b05f11cfaa] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=36.9 K 2024-11-13T18:37:41,858 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a675eace4a834a64bf35b05a7960fbf9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731523058539 2024-11-13T18:37:41,858 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca36b54dc7f43a4b90327c5cd8a1b28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731523059770 2024-11-13T18:37:41,859 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting af5a4c8135f84a32b43067b05f11cfaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731523060948 2024-11-13T18:37:41,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742118_1294 (size=13323) 2024-11-13T18:37:41,871 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/3316a37cde034a00aea09dcc2ac7ec03 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/3316a37cde034a00aea09dcc2ac7ec03 2024-11-13T18:37:41,874 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:41,875 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/6c7bc7a189154be1af9c58863f3bcc64 is 50, key is test_row_0/C:col10/1731523060948/Put/seqid=0 2024-11-13T18:37:41,876 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 3316a37cde034a00aea09dcc2ac7ec03(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:41,876 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:41,876 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=13, startTime=1731523061810; duration=0sec 2024-11-13T18:37:41,877 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:41,877 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:41,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742119_1295 (size=13323) 2024-11-13T18:37:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:41,986 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:41,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T18:37:41,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ca2d49928eeb4abf92d156f0d728215c is 50, key is test_row_0/A:col10/1731523061345/Put/seqid=0 2024-11-13T18:37:42,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742120_1296 (size=14741) 2024-11-13T18:37:42,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ca2d49928eeb4abf92d156f0d728215c 2024-11-13T18:37:42,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523122004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523122004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523122014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0 is 50, key is test_row_0/B:col10/1731523061345/Put/seqid=0 2024-11-13T18:37:42,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742121_1297 (size=12301) 2024-11-13T18:37:42,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523122114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523122114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523122123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,304 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/6c7bc7a189154be1af9c58863f3bcc64 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7bc7a189154be1af9c58863f3bcc64 2024-11-13T18:37:42,312 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into 6c7bc7a189154be1af9c58863f3bcc64(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:42,312 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:42,312 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=13, startTime=1731523061810; duration=0sec 2024-11-13T18:37:42,312 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:42,312 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:42,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523122324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523122324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523122333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0 2024-11-13T18:37:42,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/397753f691f14579865be3581659664b is 50, key is test_row_0/C:col10/1731523061345/Put/seqid=0 2024-11-13T18:37:42,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742122_1298 (size=12301) 2024-11-13T18:37:42,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523122633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523122633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523122643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T18:37:42,824 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:42,824 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-13T18:37:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:37:42,826 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:42,827 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:42,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:37:42,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43014 deadline: 1731523122934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,939 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in f8326d4028e020fa48ab6da5484ce85e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42970 deadline: 1731523122943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:42,946 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in f8326d4028e020fa48ab6da5484ce85e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:37:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:42,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/397753f691f14579865be3581659664b 2024-11-13T18:37:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ca2d49928eeb4abf92d156f0d728215c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ca2d49928eeb4abf92d156f0d728215c 2024-11-13T18:37:42,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ca2d49928eeb4abf92d156f0d728215c, entries=200, sequenceid=462, filesize=14.4 K 2024-11-13T18:37:42,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0 2024-11-13T18:37:42,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0, entries=150, sequenceid=462, filesize=12.0 K 2024-11-13T18:37:42,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/397753f691f14579865be3581659664b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/397753f691f14579865be3581659664b 2024-11-13T18:37:42,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/397753f691f14579865be3581659664b, entries=150, sequenceid=462, filesize=12.0 K 2024-11-13T18:37:42,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f8326d4028e020fa48ab6da5484ce85e in 986ms, sequenceid=462, compaction requested=false 2024-11-13T18:37:42,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:42,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T18:37:42,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:42,980 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/cbed3f9179bf4faebe8f4911b3b20a93 is 50, key is test_row_0/A:col10/1731523062005/Put/seqid=0 2024-11-13T18:37:43,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742123_1299 (size=12301) 2024-11-13T18:37:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:37:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:43,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:43,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523123200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523123204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523123204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523123313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523123315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523123315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,440 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/cbed3f9179bf4faebe8f4911b3b20a93 2024-11-13T18:37:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/07f9f564a097467a97932185f0f3e406 is 50, key is test_row_0/B:col10/1731523062005/Put/seqid=0 2024-11-13T18:37:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:37:43,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742124_1300 (size=12301) 2024-11-13T18:37:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523123524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523123525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523123525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523123833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523123833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523123843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:43,861 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/07f9f564a097467a97932185f0f3e406 2024-11-13T18:37:43,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/d7b38e359193458cbaf80e7332530bdc is 50, key is test_row_0/C:col10/1731523062005/Put/seqid=0 2024-11-13T18:37:43,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742125_1301 (size=12301) 2024-11-13T18:37:43,872 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/d7b38e359193458cbaf80e7332530bdc 2024-11-13T18:37:43,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/cbed3f9179bf4faebe8f4911b3b20a93 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cbed3f9179bf4faebe8f4911b3b20a93 2024-11-13T18:37:43,879 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cbed3f9179bf4faebe8f4911b3b20a93, entries=150, sequenceid=474, filesize=12.0 K 2024-11-13T18:37:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/07f9f564a097467a97932185f0f3e406 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/07f9f564a097467a97932185f0f3e406 2024-11-13T18:37:43,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:37:43,883 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/07f9f564a097467a97932185f0f3e406, entries=150, sequenceid=474, filesize=12.0 K 2024-11-13T18:37:43,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/d7b38e359193458cbaf80e7332530bdc as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d7b38e359193458cbaf80e7332530bdc 2024-11-13T18:37:43,887 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d7b38e359193458cbaf80e7332530bdc, entries=150, sequenceid=474, filesize=12.0 K 2024-11-13T18:37:43,888 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for f8326d4028e020fa48ab6da5484ce85e in 908ms, sequenceid=474, compaction requested=true 2024-11-13T18:37:43,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:43,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:43,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-13T18:37:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-13T18:37:43,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T18:37:43,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-11-13T18:37:43,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0680 sec 2024-11-13T18:37:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T18:37:43,963 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:43,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-13T18:37:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:37:43,965 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:43,966 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:43,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:37:44,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T18:37:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:44,119 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:37:44,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/160a833a9d7d435587b8888cd0fc9665 is 50, key is test_row_0/A:col10/1731523063196/Put/seqid=0 2024-11-13T18:37:44,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742126_1302 (size=12301) 2024-11-13T18:37:44,139 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/160a833a9d7d435587b8888cd0fc9665 2024-11-13T18:37:44,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/2d935236264c400ea5052eb2d1cd55ca is 50, key is test_row_0/B:col10/1731523063196/Put/seqid=0 2024-11-13T18:37:44,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742127_1303 (size=12301) 2024-11-13T18:37:44,160 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/2d935236264c400ea5052eb2d1cd55ca 2024-11-13T18:37:44,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0b26755ca20d4921b33c579a7675e90f is 50, key is test_row_0/C:col10/1731523063196/Put/seqid=0 2024-11-13T18:37:44,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742128_1304 (size=12301) 2024-11-13T18:37:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:37:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:44,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523124355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523124357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523124358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523124463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523124473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523124473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,574 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0b26755ca20d4921b33c579a7675e90f 2024-11-13T18:37:44,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/160a833a9d7d435587b8888cd0fc9665 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/160a833a9d7d435587b8888cd0fc9665 2024-11-13T18:37:44,589 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/160a833a9d7d435587b8888cd0fc9665, entries=150, sequenceid=500, filesize=12.0 K 2024-11-13T18:37:44,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/2d935236264c400ea5052eb2d1cd55ca as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/2d935236264c400ea5052eb2d1cd55ca 2024-11-13T18:37:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:37:44,594 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/2d935236264c400ea5052eb2d1cd55ca, entries=150, sequenceid=500, filesize=12.0 K 2024-11-13T18:37:44,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/0b26755ca20d4921b33c579a7675e90f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0b26755ca20d4921b33c579a7675e90f 2024-11-13T18:37:44,599 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0b26755ca20d4921b33c579a7675e90f, entries=150, sequenceid=500, filesize=12.0 K 2024-11-13T18:37:44,600 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f8326d4028e020fa48ab6da5484ce85e in 481ms, sequenceid=500, compaction requested=true 2024-11-13T18:37:44,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:44,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:44,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-13T18:37:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-13T18:37:44,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T18:37:44,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 636 msec 2024-11-13T18:37:44,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 640 msec 2024-11-13T18:37:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:44,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:44,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:37:44,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/cdf054cf8d4a49ffad253f1c425bac6a is 50, key is test_row_0/A:col10/1731523064357/Put/seqid=0 2024-11-13T18:37:44,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742129_1305 (size=14741) 2024-11-13T18:37:44,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/cdf054cf8d4a49ffad253f1c425bac6a 2024-11-13T18:37:44,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/a910370df7ee41d386ffe84025445ffb is 50, key is test_row_0/B:col10/1731523064357/Put/seqid=0 2024-11-13T18:37:44,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742130_1306 (size=12301) 2024-11-13T18:37:44,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523124732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523124734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523124737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523124843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523124843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523124843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523125053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523125053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523125054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T18:37:45,103 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:45,104 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-13T18:37:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:37:45,106 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:45,107 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:45,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:45,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/a910370df7ee41d386ffe84025445ffb 2024-11-13T18:37:45,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/1c29ae6b8e734e0badc695d074dfa5a7 is 50, key is test_row_0/C:col10/1731523064357/Put/seqid=0 2024-11-13T18:37:45,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742131_1307 (size=12301) 2024-11-13T18:37:45,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/1c29ae6b8e734e0badc695d074dfa5a7 2024-11-13T18:37:45,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/cdf054cf8d4a49ffad253f1c425bac6a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cdf054cf8d4a49ffad253f1c425bac6a 2024-11-13T18:37:45,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cdf054cf8d4a49ffad253f1c425bac6a, entries=200, sequenceid=511, filesize=14.4 K 2024-11-13T18:37:45,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/a910370df7ee41d386ffe84025445ffb as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/a910370df7ee41d386ffe84025445ffb 2024-11-13T18:37:45,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/a910370df7ee41d386ffe84025445ffb, entries=150, sequenceid=511, filesize=12.0 K 2024-11-13T18:37:45,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/1c29ae6b8e734e0badc695d074dfa5a7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1c29ae6b8e734e0badc695d074dfa5a7 2024-11-13T18:37:45,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1c29ae6b8e734e0badc695d074dfa5a7, entries=150, sequenceid=511, filesize=12.0 K 2024-11-13T18:37:45,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f8326d4028e020fa48ab6da5484ce85e in 503ms, sequenceid=511, compaction requested=true 2024-11-13T18:37:45,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:45,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:45,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:45,180 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T18:37:45,180 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T18:37:45,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:45,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:45,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:45,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:45,182 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62527 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T18:37:45,182 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:45,182 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:45,182 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/3316a37cde034a00aea09dcc2ac7ec03, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/07f9f564a097467a97932185f0f3e406, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/2d935236264c400ea5052eb2d1cd55ca, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/a910370df7ee41d386ffe84025445ffb] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=61.1 K 2024-11-13T18:37:45,182 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67407 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T18:37:45,182 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:45,182 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:45,183 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c20e5ed0d34e43379eadb267178146f3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ca2d49928eeb4abf92d156f0d728215c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cbed3f9179bf4faebe8f4911b3b20a93, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/160a833a9d7d435587b8888cd0fc9665, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cdf054cf8d4a49ffad253f1c425bac6a] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=65.8 K 2024-11-13T18:37:45,183 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3316a37cde034a00aea09dcc2ac7ec03, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731523060948 2024-11-13T18:37:45,183 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c20e5ed0d34e43379eadb267178146f3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731523060948 2024-11-13T18:37:45,184 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a0fdf9c8e2d4fa19d8adae6ec8fffb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731523061345 2024-11-13T18:37:45,184 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2d49928eeb4abf92d156f0d728215c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731523061345 2024-11-13T18:37:45,184 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 07f9f564a097467a97932185f0f3e406, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731523062002 2024-11-13T18:37:45,185 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbed3f9179bf4faebe8f4911b3b20a93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731523062002 2024-11-13T18:37:45,185 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d935236264c400ea5052eb2d1cd55ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731523063196 2024-11-13T18:37:45,185 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 160a833a9d7d435587b8888cd0fc9665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731523063196 2024-11-13T18:37:45,185 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a910370df7ee41d386ffe84025445ffb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731523064346 2024-11-13T18:37:45,185 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf054cf8d4a49ffad253f1c425bac6a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731523064346 2024-11-13T18:37:45,198 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:45,199 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/4693d94dbccb49d8b35c5c7478508273 is 50, key is test_row_0/B:col10/1731523064357/Put/seqid=0 2024-11-13T18:37:45,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742132_1308 (size=13493) 2024-11-13T18:37:45,209 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:45,210 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ceb5cb6cb4694cbc8811cffc3f851049 is 50, key is test_row_0/A:col10/1731523064357/Put/seqid=0 2024-11-13T18:37:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:37:45,221 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/4693d94dbccb49d8b35c5c7478508273 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/4693d94dbccb49d8b35c5c7478508273 2024-11-13T18:37:45,227 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 4693d94dbccb49d8b35c5c7478508273(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:45,227 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:45,227 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=11, startTime=1731523065180; duration=0sec 2024-11-13T18:37:45,227 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:45,227 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:45,227 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T18:37:45,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742133_1309 (size=13493) 2024-11-13T18:37:45,229 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62527 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T18:37:45,229 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:45,229 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:45,229 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7bc7a189154be1af9c58863f3bcc64, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/397753f691f14579865be3581659664b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d7b38e359193458cbaf80e7332530bdc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0b26755ca20d4921b33c579a7675e90f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1c29ae6b8e734e0badc695d074dfa5a7] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=61.1 K 2024-11-13T18:37:45,230 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c7bc7a189154be1af9c58863f3bcc64, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731523060948 2024-11-13T18:37:45,230 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 397753f691f14579865be3581659664b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731523061345 2024-11-13T18:37:45,235 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b38e359193458cbaf80e7332530bdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731523062002 2024-11-13T18:37:45,235 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b26755ca20d4921b33c579a7675e90f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731523063196 2024-11-13T18:37:45,235 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c29ae6b8e734e0badc695d074dfa5a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731523064346 2024-11-13T18:37:45,246 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:45,247 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/4615a8d245504f6da33f82addcf84ce8 is 50, key is test_row_0/C:col10/1731523064357/Put/seqid=0 2024-11-13T18:37:45,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T18:37:45,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:45,261 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:37:45,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742134_1310 (size=13493) 2024-11-13T18:37:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b2826e468770442a810ac975140c81ab is 50, key is test_row_0/A:col10/1731523064735/Put/seqid=0 2024-11-13T18:37:45,280 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/4615a8d245504f6da33f82addcf84ce8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4615a8d245504f6da33f82addcf84ce8 2024-11-13T18:37:45,286 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into 4615a8d245504f6da33f82addcf84ce8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:45,286 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:45,286 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=11, startTime=1731523065180; duration=0sec 2024-11-13T18:37:45,286 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:45,286 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:45,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742135_1311 (size=12301) 2024-11-13T18:37:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:45,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. as already flushing 2024-11-13T18:37:45,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523125381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523125381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523125383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:37:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523125494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523125494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523125494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,633 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/ceb5cb6cb4694cbc8811cffc3f851049 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ceb5cb6cb4694cbc8811cffc3f851049 2024-11-13T18:37:45,637 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into ceb5cb6cb4694cbc8811cffc3f851049(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:45,637 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:45,637 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=11, startTime=1731523065180; duration=0sec 2024-11-13T18:37:45,637 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:45,637 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:45,690 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b2826e468770442a810ac975140c81ab 2024-11-13T18:37:45,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/7e7c9460f623416b80bf18112c7aeddd is 50, key is test_row_0/B:col10/1731523064735/Put/seqid=0 2024-11-13T18:37:45,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742136_1312 (size=12301) 2024-11-13T18:37:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523125703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523125704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523125704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:37:45,792 INFO [Thread-982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-982. 2024-11-13T18:37:45,792 DEBUG [Thread-982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:45,792 DEBUG [Thread-982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,792 DEBUG [Thread-982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:45,793 INFO [Thread-984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-984. 2024-11-13T18:37:45,793 DEBUG [Thread-984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:45,793 DEBUG [Thread-984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,793 DEBUG [Thread-984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:45,794 INFO [Thread-986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-986. 2024-11-13T18:37:45,794 DEBUG [Thread-986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:45,794 DEBUG [Thread-986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,794 DEBUG [Thread-986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:45,795 INFO [Thread-988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-988. 2024-11-13T18:37:45,795 DEBUG [Thread-988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:45,795 DEBUG [Thread-988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,795 DEBUG [Thread-988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:45,796 INFO [Thread-980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-980. 2024-11-13T18:37:45,796 DEBUG [Thread-980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:45,796 DEBUG [Thread-980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,796 DEBUG [Thread-980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:45,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:46,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:46,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:46,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523126013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523126013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523126013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:46,103 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/7e7c9460f623416b80bf18112c7aeddd 2024-11-13T18:37:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/31dea4827e7e44e8a596376f3afba8ea is 50, key is test_row_0/C:col10/1731523064735/Put/seqid=0 2024-11-13T18:37:46,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742137_1313 (size=12301) 2024-11-13T18:37:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:37:46,513 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/31dea4827e7e44e8a596376f3afba8ea 2024-11-13T18:37:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/b2826e468770442a810ac975140c81ab as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b2826e468770442a810ac975140c81ab 2024-11-13T18:37:46,519 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b2826e468770442a810ac975140c81ab, entries=150, sequenceid=537, filesize=12.0 K 2024-11-13T18:37:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/7e7c9460f623416b80bf18112c7aeddd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7e7c9460f623416b80bf18112c7aeddd 2024-11-13T18:37:46,522 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7e7c9460f623416b80bf18112c7aeddd, entries=150, sequenceid=537, filesize=12.0 K 2024-11-13T18:37:46,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:46,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/31dea4827e7e44e8a596376f3afba8ea as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/31dea4827e7e44e8a596376f3afba8ea 2024-11-13T18:37:46,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42952 deadline: 1731523126523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42936 deadline: 1731523126523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42994 deadline: 1731523126523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:37:46,526 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/31dea4827e7e44e8a596376f3afba8ea, entries=150, sequenceid=537, filesize=12.0 K 2024-11-13T18:37:46,527 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f8326d4028e020fa48ab6da5484ce85e in 1266ms, sequenceid=537, compaction requested=false 2024-11-13T18:37:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-13T18:37:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-13T18:37:46,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-13T18:37:46,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-11-13T18:37:46,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4260 sec 2024-11-13T18:37:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T18:37:47,254 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:47,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:47,536 INFO [Thread-971 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-971. 2024-11-13T18:37:47,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T18:37:47,536 DEBUG [Thread-971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:47,536 INFO [Thread-969 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-969. 2024-11-13T18:37:47,536 INFO [Thread-977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-977. 2024-11-13T18:37:47,536 DEBUG [Thread-971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:47,536 DEBUG [Thread-977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:47,536 DEBUG [Thread-969 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:47,536 DEBUG [Thread-977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:47,536 DEBUG [Thread-969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:47,536 DEBUG [Thread-971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:47,536 DEBUG [Thread-969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:47,536 DEBUG [Thread-977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:47,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:47,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:47,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:47,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/dd30038174b846bfba0da0de07309b6f is 50, key is test_row_0/A:col10/1731523065380/Put/seqid=0 2024-11-13T18:37:47,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742138_1314 (size=12301) 2024-11-13T18:37:47,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/dd30038174b846bfba0da0de07309b6f 2024-11-13T18:37:47,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/e003b9f9f7264283b53d22f52c89d3d7 is 50, key is test_row_0/B:col10/1731523065380/Put/seqid=0 2024-11-13T18:37:47,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742139_1315 (size=12301) 2024-11-13T18:37:48,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/e003b9f9f7264283b53d22f52c89d3d7 2024-11-13T18:37:48,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/1e24591fb97d4cb29131b4b5fba98136 is 50, key is test_row_0/C:col10/1731523065380/Put/seqid=0 2024-11-13T18:37:48,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742140_1316 (size=12301) 2024-11-13T18:37:48,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/1e24591fb97d4cb29131b4b5fba98136 2024-11-13T18:37:48,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/dd30038174b846bfba0da0de07309b6f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/dd30038174b846bfba0da0de07309b6f 2024-11-13T18:37:48,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/dd30038174b846bfba0da0de07309b6f, entries=150, sequenceid=553, filesize=12.0 K 2024-11-13T18:37:48,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/e003b9f9f7264283b53d22f52c89d3d7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e003b9f9f7264283b53d22f52c89d3d7 2024-11-13T18:37:48,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e003b9f9f7264283b53d22f52c89d3d7, entries=150, sequenceid=553, filesize=12.0 K 2024-11-13T18:37:48,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/1e24591fb97d4cb29131b4b5fba98136 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1e24591fb97d4cb29131b4b5fba98136 2024-11-13T18:37:48,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1e24591fb97d4cb29131b4b5fba98136, entries=150, sequenceid=553, filesize=12.0 K 2024-11-13T18:37:48,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for f8326d4028e020fa48ab6da5484ce85e in 1247ms, sequenceid=553, compaction requested=true 2024-11-13T18:37:48,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:48,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:37:48,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:48,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:37:48,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:48,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8326d4028e020fa48ab6da5484ce85e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:37:48,783 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:48,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:48,783 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:48,783 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:48,783 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:48,783 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/A is initiating minor compaction (all files) 2024-11-13T18:37:48,783 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/B is initiating minor compaction (all files) 2024-11-13T18:37:48,783 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/A in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:48,783 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/B in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:48,784 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ceb5cb6cb4694cbc8811cffc3f851049, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b2826e468770442a810ac975140c81ab, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/dd30038174b846bfba0da0de07309b6f] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=37.2 K 2024-11-13T18:37:48,784 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/4693d94dbccb49d8b35c5c7478508273, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7e7c9460f623416b80bf18112c7aeddd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e003b9f9f7264283b53d22f52c89d3d7] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=37.2 K 2024-11-13T18:37:48,784 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4693d94dbccb49d8b35c5c7478508273, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731523064346 2024-11-13T18:37:48,784 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceb5cb6cb4694cbc8811cffc3f851049, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731523064346 2024-11-13T18:37:48,784 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2826e468770442a810ac975140c81ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731523064722 2024-11-13T18:37:48,784 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e7c9460f623416b80bf18112c7aeddd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731523064722 2024-11-13T18:37:48,784 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd30038174b846bfba0da0de07309b6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1731523065380 2024-11-13T18:37:48,784 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e003b9f9f7264283b53d22f52c89d3d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1731523065380 2024-11-13T18:37:48,792 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#A#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:48,792 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:48,793 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0fed8fc26c4b4fdda1a0254dbada3616 is 50, key is test_row_0/B:col10/1731523065380/Put/seqid=0 2024-11-13T18:37:48,793 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/f9ca8f5c69ae47d89f02e7788888c5c5 is 50, key is test_row_0/A:col10/1731523065380/Put/seqid=0 2024-11-13T18:37:48,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742141_1317 (size=13595) 2024-11-13T18:37:48,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742142_1318 (size=13595) 2024-11-13T18:37:49,203 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0fed8fc26c4b4fdda1a0254dbada3616 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fed8fc26c4b4fdda1a0254dbada3616 2024-11-13T18:37:49,203 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/f9ca8f5c69ae47d89f02e7788888c5c5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f9ca8f5c69ae47d89f02e7788888c5c5 2024-11-13T18:37:49,207 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/A of f8326d4028e020fa48ab6da5484ce85e into f9ca8f5c69ae47d89f02e7788888c5c5(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:49,207 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/B of f8326d4028e020fa48ab6da5484ce85e into 0fed8fc26c4b4fdda1a0254dbada3616(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:49,207 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:49,207 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:49,207 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/B, priority=13, startTime=1731523068782; duration=0sec 2024-11-13T18:37:49,207 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/A, priority=13, startTime=1731523068782; duration=0sec 2024-11-13T18:37:49,207 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:37:49,207 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:B 2024-11-13T18:37:49,207 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:49,207 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:A 2024-11-13T18:37:49,207 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:37:49,208 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:37:49,208 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): f8326d4028e020fa48ab6da5484ce85e/C is initiating minor compaction (all files) 2024-11-13T18:37:49,208 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8326d4028e020fa48ab6da5484ce85e/C in TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:49,208 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4615a8d245504f6da33f82addcf84ce8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/31dea4827e7e44e8a596376f3afba8ea, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1e24591fb97d4cb29131b4b5fba98136] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp, totalSize=37.2 K 2024-11-13T18:37:49,208 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4615a8d245504f6da33f82addcf84ce8, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731523064346 2024-11-13T18:37:49,209 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 31dea4827e7e44e8a596376f3afba8ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731523064722 2024-11-13T18:37:49,209 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e24591fb97d4cb29131b4b5fba98136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1731523065380 2024-11-13T18:37:49,215 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8326d4028e020fa48ab6da5484ce85e#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:37:49,215 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/67e7efe6ce674511b8b414225b7bcb08 is 50, key is test_row_0/C:col10/1731523065380/Put/seqid=0 2024-11-13T18:37:49,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742143_1319 (size=13595) 2024-11-13T18:37:49,627 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/67e7efe6ce674511b8b414225b7bcb08 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/67e7efe6ce674511b8b414225b7bcb08 2024-11-13T18:37:49,632 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8326d4028e020fa48ab6da5484ce85e/C of f8326d4028e020fa48ab6da5484ce85e into 67e7efe6ce674511b8b414225b7bcb08(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:37:49,632 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8326d4028e020fa48ab6da5484ce85e: 2024-11-13T18:37:49,632 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., storeName=f8326d4028e020fa48ab6da5484ce85e/C, priority=13, startTime=1731523068782; duration=0sec 2024-11-13T18:37:49,632 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:37:49,632 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8326d4028e020fa48ab6da5484ce85e:C 2024-11-13T18:37:52,975 INFO [Thread-973 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-973. 2024-11-13T18:37:52,975 DEBUG [Thread-973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:52,975 DEBUG [Thread-973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:52,975 DEBUG [Thread-973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:52,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:53,005 INFO [Thread-975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-975. 2024-11-13T18:37:53,005 DEBUG [Thread-975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:37:53,005 DEBUG [Thread-975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:53,006 DEBUG [Thread-975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:53,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2436 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2399 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2425 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2452 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2413 2024-11-13T18:37:53,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:37:53,006 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:37:53,006 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:37:53,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:53,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:53,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:37:53,009 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:37:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T18:37:53,017 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523073017"}]},"ts":"1731523073017"} 2024-11-13T18:37:53,020 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:37:53,020 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:37:53,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:37:53,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, UNASSIGN}] 2024-11-13T18:37:53,024 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, UNASSIGN 2024-11-13T18:37:53,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f8326d4028e020fa48ab6da5484ce85e, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:53,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, UNASSIGN because future has completed 2024-11-13T18:37:53,033 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:37:53,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:37:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T18:37:53,186 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:53,186 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:37:53,186 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing f8326d4028e020fa48ab6da5484ce85e, disabling compactions & flushes 2024-11-13T18:37:53,186 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:53,186 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:53,187 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. after waiting 0 ms 2024-11-13T18:37:53,187 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:53,187 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing f8326d4028e020fa48ab6da5484ce85e 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-13T18:37:53,192 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/2d29b001784b4eabb671202912116bd4 is 50, key is test_row_0/A:col10/1731523073004/Put/seqid=0 2024-11-13T18:37:53,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742144_1320 (size=9857) 2024-11-13T18:37:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T18:37:53,603 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/2d29b001784b4eabb671202912116bd4 2024-11-13T18:37:53,613 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0105aae61131487b984755b7e98a433e is 50, key is test_row_0/B:col10/1731523073004/Put/seqid=0 2024-11-13T18:37:53,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742145_1321 (size=9857) 2024-11-13T18:37:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T18:37:54,018 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0105aae61131487b984755b7e98a433e 2024-11-13T18:37:54,025 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/26759e6f1ca84a6a9db72d1c2fb7acfd is 50, key is test_row_0/C:col10/1731523073004/Put/seqid=0 2024-11-13T18:37:54,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742146_1322 (size=9857) 2024-11-13T18:37:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T18:37:54,429 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/26759e6f1ca84a6a9db72d1c2fb7acfd 2024-11-13T18:37:54,435 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/A/2d29b001784b4eabb671202912116bd4 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d29b001784b4eabb671202912116bd4 2024-11-13T18:37:54,439 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d29b001784b4eabb671202912116bd4, entries=100, sequenceid=561, filesize=9.6 K 2024-11-13T18:37:54,440 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/B/0105aae61131487b984755b7e98a433e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0105aae61131487b984755b7e98a433e 2024-11-13T18:37:54,445 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0105aae61131487b984755b7e98a433e, entries=100, sequenceid=561, filesize=9.6 K 2024-11-13T18:37:54,447 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/.tmp/C/26759e6f1ca84a6a9db72d1c2fb7acfd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/26759e6f1ca84a6a9db72d1c2fb7acfd 2024-11-13T18:37:54,454 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/26759e6f1ca84a6a9db72d1c2fb7acfd, entries=100, sequenceid=561, filesize=9.6 K 2024-11-13T18:37:54,455 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for f8326d4028e020fa48ab6da5484ce85e in 1268ms, sequenceid=561, compaction requested=false 2024-11-13T18:37:54,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/308a9c4773b74187ac2c3b49db46bb73, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ecfb60e1540c49b391d94bb06094b388, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e3e25d5da6e3437091ca0dd206023b4b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6ac3e48a2855464a824ad7145275bdda, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/fb6631fff2024d6ea95bb8bcce78929e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c19e6282565a44a8b54198c44818330b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/a21bd90502ea4551aba8748bc4d1556f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/54ada0444ff54e3a94963d99bf65376f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/485044fc036a488795118a9593d8ed24, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/47f7b327bda84314a2ebc7d7e3c7b5e7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b0d4dae661d74a4e85d41059676a43ec, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d5ded9d43b445a99162d0e49498e9dc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/52f84a103a7f4a67849c467c2f0e82ef, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f090432c4a3d470c989237bc3e8b5130, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f6bec67c6a1247a0857119a6ede90300, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/60e5ebd88dbb4840b9a1723ba09f9aa3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/8056ee98b57c406982c52615d836342b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ba20cf6ec96a4cfb9974773c6000e20c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/86e331c122824f9f89d0ea6dc652c9d6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/7cb733d7a80046e395b34db3f7d6bd5e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e928cba5fce24ff4b6c6ca9cfe90c790, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/408d9d03aaa44ff2b96e3581a91dca5c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b14747f41d3a476186ad0b940b92927a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b45a2cc79fc4430a8303244be81f91aa, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/96f4dd116eec4e85bdf1f043e911da19, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/840b4da3840647b5b7d742aa49e1e0a9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/4cbf7b058c874744a0a60e5f2c1cfc4e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/0eeebbd32f9b408bbd6ccd4db949884a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6a5ea0e0020c4590a0017c532ce2f6c6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c0908c00453040e1afe7eea4a288b25b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c20e5ed0d34e43379eadb267178146f3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ca2d49928eeb4abf92d156f0d728215c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cbed3f9179bf4faebe8f4911b3b20a93, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/160a833a9d7d435587b8888cd0fc9665, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cdf054cf8d4a49ffad253f1c425bac6a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ceb5cb6cb4694cbc8811cffc3f851049, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b2826e468770442a810ac975140c81ab, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/dd30038174b846bfba0da0de07309b6f] to archive 2024-11-13T18:37:54,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:37:54,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/308a9c4773b74187ac2c3b49db46bb73 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/308a9c4773b74187ac2c3b49db46bb73 2024-11-13T18:37:54,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ecfb60e1540c49b391d94bb06094b388 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ecfb60e1540c49b391d94bb06094b388 2024-11-13T18:37:54,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e3e25d5da6e3437091ca0dd206023b4b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e3e25d5da6e3437091ca0dd206023b4b 2024-11-13T18:37:54,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6ac3e48a2855464a824ad7145275bdda to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6ac3e48a2855464a824ad7145275bdda 2024-11-13T18:37:54,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/fb6631fff2024d6ea95bb8bcce78929e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/fb6631fff2024d6ea95bb8bcce78929e 2024-11-13T18:37:54,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c19e6282565a44a8b54198c44818330b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c19e6282565a44a8b54198c44818330b 2024-11-13T18:37:54,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/a21bd90502ea4551aba8748bc4d1556f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/a21bd90502ea4551aba8748bc4d1556f 2024-11-13T18:37:54,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/54ada0444ff54e3a94963d99bf65376f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/54ada0444ff54e3a94963d99bf65376f 2024-11-13T18:37:54,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/485044fc036a488795118a9593d8ed24 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/485044fc036a488795118a9593d8ed24 2024-11-13T18:37:54,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/47f7b327bda84314a2ebc7d7e3c7b5e7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/47f7b327bda84314a2ebc7d7e3c7b5e7 2024-11-13T18:37:54,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b0d4dae661d74a4e85d41059676a43ec to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b0d4dae661d74a4e85d41059676a43ec 2024-11-13T18:37:54,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d5ded9d43b445a99162d0e49498e9dc to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d5ded9d43b445a99162d0e49498e9dc 2024-11-13T18:37:54,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/52f84a103a7f4a67849c467c2f0e82ef to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/52f84a103a7f4a67849c467c2f0e82ef 2024-11-13T18:37:54,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f090432c4a3d470c989237bc3e8b5130 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f090432c4a3d470c989237bc3e8b5130 2024-11-13T18:37:54,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f6bec67c6a1247a0857119a6ede90300 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f6bec67c6a1247a0857119a6ede90300 2024-11-13T18:37:54,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/60e5ebd88dbb4840b9a1723ba09f9aa3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/60e5ebd88dbb4840b9a1723ba09f9aa3 2024-11-13T18:37:54,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/8056ee98b57c406982c52615d836342b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/8056ee98b57c406982c52615d836342b 2024-11-13T18:37:54,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ba20cf6ec96a4cfb9974773c6000e20c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ba20cf6ec96a4cfb9974773c6000e20c 2024-11-13T18:37:54,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/86e331c122824f9f89d0ea6dc652c9d6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/86e331c122824f9f89d0ea6dc652c9d6 2024-11-13T18:37:54,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/7cb733d7a80046e395b34db3f7d6bd5e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/7cb733d7a80046e395b34db3f7d6bd5e 2024-11-13T18:37:54,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e928cba5fce24ff4b6c6ca9cfe90c790 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/e928cba5fce24ff4b6c6ca9cfe90c790 2024-11-13T18:37:54,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/408d9d03aaa44ff2b96e3581a91dca5c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/408d9d03aaa44ff2b96e3581a91dca5c 2024-11-13T18:37:54,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b14747f41d3a476186ad0b940b92927a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b14747f41d3a476186ad0b940b92927a 2024-11-13T18:37:54,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b45a2cc79fc4430a8303244be81f91aa to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b45a2cc79fc4430a8303244be81f91aa 2024-11-13T18:37:54,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/96f4dd116eec4e85bdf1f043e911da19 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/96f4dd116eec4e85bdf1f043e911da19 2024-11-13T18:37:54,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/840b4da3840647b5b7d742aa49e1e0a9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/840b4da3840647b5b7d742aa49e1e0a9 2024-11-13T18:37:54,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/4cbf7b058c874744a0a60e5f2c1cfc4e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/4cbf7b058c874744a0a60e5f2c1cfc4e 2024-11-13T18:37:54,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/0eeebbd32f9b408bbd6ccd4db949884a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/0eeebbd32f9b408bbd6ccd4db949884a 2024-11-13T18:37:54,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6a5ea0e0020c4590a0017c532ce2f6c6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/6a5ea0e0020c4590a0017c532ce2f6c6 2024-11-13T18:37:54,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c0908c00453040e1afe7eea4a288b25b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c0908c00453040e1afe7eea4a288b25b 2024-11-13T18:37:54,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c20e5ed0d34e43379eadb267178146f3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/c20e5ed0d34e43379eadb267178146f3 2024-11-13T18:37:54,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ca2d49928eeb4abf92d156f0d728215c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ca2d49928eeb4abf92d156f0d728215c 2024-11-13T18:37:54,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cbed3f9179bf4faebe8f4911b3b20a93 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cbed3f9179bf4faebe8f4911b3b20a93 2024-11-13T18:37:54,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/160a833a9d7d435587b8888cd0fc9665 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/160a833a9d7d435587b8888cd0fc9665 2024-11-13T18:37:54,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cdf054cf8d4a49ffad253f1c425bac6a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/cdf054cf8d4a49ffad253f1c425bac6a 2024-11-13T18:37:54,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ceb5cb6cb4694cbc8811cffc3f851049 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/ceb5cb6cb4694cbc8811cffc3f851049 2024-11-13T18:37:54,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b2826e468770442a810ac975140c81ab to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/b2826e468770442a810ac975140c81ab 2024-11-13T18:37:54,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/dd30038174b846bfba0da0de07309b6f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/dd30038174b846bfba0da0de07309b6f 2024-11-13T18:37:54,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fb2f663670b4f55a65a66fea49d932f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c358bb5d60264c8b9ffc5ba654a6cff1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/01fb61d679464ab78a17c6924127139f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/84164f8284e5463a99fb1e022a441673, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9ffb4a8d927545c9a6844946567259a6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/47279ecadc79434982d39454243902c7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ba40d3b8df604360b5a3eba15d62294f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/fe37cc97270a4bc68f9c036d412205dd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7a55d7c3ad974df9bed37921e103e95a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5b58b01dbdd64a8999bd44ce6dd3d6b7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/d07ab75fdf10434db5610308a01e8838, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/73fa57761ed24d3292984bb6aefe6961, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0912f5fcf75a42698546cb8ed63d9636, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e5e631a37360438286d2855cace8b4c9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/274434554b3b4edfa67d9eed6346d60f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9f446f6018ba41ad960c5292126bffe1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/6979842875574e54ba30c2626c58f706, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/64d5b3c2ced846d9a4da8b589d251839, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9da8a372c2ab48e69eda766164071a8f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5783c12a48f640ea9cd853c87cca6d54, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/19d813468ded4daaa3aa677adeee38ff, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/1ced747f894441bebfebfeece83b0ef6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ea0aff93e94e4104ae30571202480187, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/b49cfa6615b84a439629f96de118e7ee, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0a16d2541427463c819a4e44361e1a61, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/67fa9b09a94d4527bcbca42828a2ab34, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0c8784ad59024191bdbf00bc43578216, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/93f8c6054408431e94c80ae835a2015c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c1cea3ffd32940119b055ad645f9890c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/3316a37cde034a00aea09dcc2ac7ec03, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ee59adfe525946e28716d2ba6a4d3e1e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/07f9f564a097467a97932185f0f3e406, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/2d935236264c400ea5052eb2d1cd55ca, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/4693d94dbccb49d8b35c5c7478508273, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/a910370df7ee41d386ffe84025445ffb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7e7c9460f623416b80bf18112c7aeddd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e003b9f9f7264283b53d22f52c89d3d7] to archive 2024-11-13T18:37:54,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:37:54,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fb2f663670b4f55a65a66fea49d932f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fb2f663670b4f55a65a66fea49d932f 2024-11-13T18:37:54,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c358bb5d60264c8b9ffc5ba654a6cff1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c358bb5d60264c8b9ffc5ba654a6cff1 2024-11-13T18:37:54,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/01fb61d679464ab78a17c6924127139f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/01fb61d679464ab78a17c6924127139f 2024-11-13T18:37:54,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/84164f8284e5463a99fb1e022a441673 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/84164f8284e5463a99fb1e022a441673 2024-11-13T18:37:54,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9ffb4a8d927545c9a6844946567259a6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9ffb4a8d927545c9a6844946567259a6 2024-11-13T18:37:54,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/47279ecadc79434982d39454243902c7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/47279ecadc79434982d39454243902c7 2024-11-13T18:37:54,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ba40d3b8df604360b5a3eba15d62294f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ba40d3b8df604360b5a3eba15d62294f 2024-11-13T18:37:54,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/fe37cc97270a4bc68f9c036d412205dd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/fe37cc97270a4bc68f9c036d412205dd 2024-11-13T18:37:54,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7a55d7c3ad974df9bed37921e103e95a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7a55d7c3ad974df9bed37921e103e95a 2024-11-13T18:37:54,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5b58b01dbdd64a8999bd44ce6dd3d6b7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5b58b01dbdd64a8999bd44ce6dd3d6b7 2024-11-13T18:37:54,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/d07ab75fdf10434db5610308a01e8838 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/d07ab75fdf10434db5610308a01e8838 2024-11-13T18:37:54,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/73fa57761ed24d3292984bb6aefe6961 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/73fa57761ed24d3292984bb6aefe6961 2024-11-13T18:37:54,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0912f5fcf75a42698546cb8ed63d9636 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0912f5fcf75a42698546cb8ed63d9636 2024-11-13T18:37:54,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e5e631a37360438286d2855cace8b4c9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e5e631a37360438286d2855cace8b4c9 2024-11-13T18:37:54,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/274434554b3b4edfa67d9eed6346d60f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/274434554b3b4edfa67d9eed6346d60f 2024-11-13T18:37:54,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9f446f6018ba41ad960c5292126bffe1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9f446f6018ba41ad960c5292126bffe1 2024-11-13T18:37:54,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/6979842875574e54ba30c2626c58f706 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/6979842875574e54ba30c2626c58f706 2024-11-13T18:37:54,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/64d5b3c2ced846d9a4da8b589d251839 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/64d5b3c2ced846d9a4da8b589d251839 2024-11-13T18:37:54,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9da8a372c2ab48e69eda766164071a8f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/9da8a372c2ab48e69eda766164071a8f 2024-11-13T18:37:54,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5783c12a48f640ea9cd853c87cca6d54 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5783c12a48f640ea9cd853c87cca6d54 2024-11-13T18:37:54,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/19d813468ded4daaa3aa677adeee38ff to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/19d813468ded4daaa3aa677adeee38ff 2024-11-13T18:37:54,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/1ced747f894441bebfebfeece83b0ef6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/1ced747f894441bebfebfeece83b0ef6 2024-11-13T18:37:54,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ea0aff93e94e4104ae30571202480187 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ea0aff93e94e4104ae30571202480187 2024-11-13T18:37:54,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/b49cfa6615b84a439629f96de118e7ee to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/b49cfa6615b84a439629f96de118e7ee 2024-11-13T18:37:54,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0a16d2541427463c819a4e44361e1a61 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0a16d2541427463c819a4e44361e1a61 2024-11-13T18:37:54,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/67fa9b09a94d4527bcbca42828a2ab34 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/67fa9b09a94d4527bcbca42828a2ab34 2024-11-13T18:37:54,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0c8784ad59024191bdbf00bc43578216 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0c8784ad59024191bdbf00bc43578216 2024-11-13T18:37:54,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/93f8c6054408431e94c80ae835a2015c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/93f8c6054408431e94c80ae835a2015c 2024-11-13T18:37:54,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c1cea3ffd32940119b055ad645f9890c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/c1cea3ffd32940119b055ad645f9890c 2024-11-13T18:37:54,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/3316a37cde034a00aea09dcc2ac7ec03 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/3316a37cde034a00aea09dcc2ac7ec03 2024-11-13T18:37:54,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ee59adfe525946e28716d2ba6a4d3e1e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/ee59adfe525946e28716d2ba6a4d3e1e 2024-11-13T18:37:54,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/5a0fdf9c8e2d4fa19d8adae6ec8fffb0 2024-11-13T18:37:54,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/07f9f564a097467a97932185f0f3e406 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/07f9f564a097467a97932185f0f3e406 2024-11-13T18:37:54,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/2d935236264c400ea5052eb2d1cd55ca to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/2d935236264c400ea5052eb2d1cd55ca 2024-11-13T18:37:54,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/4693d94dbccb49d8b35c5c7478508273 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/4693d94dbccb49d8b35c5c7478508273 2024-11-13T18:37:54,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/a910370df7ee41d386ffe84025445ffb to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/a910370df7ee41d386ffe84025445ffb 2024-11-13T18:37:54,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7e7c9460f623416b80bf18112c7aeddd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/7e7c9460f623416b80bf18112c7aeddd 2024-11-13T18:37:54,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e003b9f9f7264283b53d22f52c89d3d7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/e003b9f9f7264283b53d22f52c89d3d7 2024-11-13T18:37:54,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/991567b1a37649cc82c3c7fb36f5935b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/78a4556b113a412d8b1611b66a49799a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/c1ba8483b5fa418db0e9b057237700ed, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/bc19d36c8dec4d1fa6a825ce62960284, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0fb3caed9277459cb165a7e787fd9bb9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5f004306f1bd40ba92b6d24b330af1ce, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e8c8360c0fa0476a9ff08d1940a633b9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/208f78b8bda64d719bc5189a04945766, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7ca4dc7b2ed14009b22f6978d7178830, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/ae1f5b8f9b3b45779c587a3741fc1e48, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d62309badfad4c279e31d6791417a18a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/8165115bf660443e9e68bd0ef61cb072, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/757356d034744efb9a802372256a1937, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/9893e57b63fd4a25a9216135132e275a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7f1ad8fa4bcc40f9951f13472057fdb9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7109e2640e4f4eab276474e4fa0aea, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4aac26c722664d088259115c439b72f4, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e496171c5d1543419661cd0c352aa1e4, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fd4d493990504072ba3e248dbe530c93, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a2a4966ef164ed483bf2659b13ee60f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6e5128428d484b969297bd1e8919eab6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4765b6363fa346789d6ed53ac6611b22, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/2b102683097b42beb549c64b036b9d17, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/da7a4cfeff9b433e970335c3c145aaf8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5d5de00f2a474a76b6c1a8c260b01470, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a35dc536d4b4a8eb7d683a3dce53579, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/a675eace4a834a64bf35b05a7960fbf9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/3b063fc89f3e40ba91fbae4926120b6e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fca36b54dc7f43a4b90327c5cd8a1b28, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7bc7a189154be1af9c58863f3bcc64, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/af5a4c8135f84a32b43067b05f11cfaa, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/397753f691f14579865be3581659664b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d7b38e359193458cbaf80e7332530bdc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0b26755ca20d4921b33c579a7675e90f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4615a8d245504f6da33f82addcf84ce8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1c29ae6b8e734e0badc695d074dfa5a7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/31dea4827e7e44e8a596376f3afba8ea, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1e24591fb97d4cb29131b4b5fba98136] to archive 2024-11-13T18:37:54,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:37:54,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/991567b1a37649cc82c3c7fb36f5935b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/991567b1a37649cc82c3c7fb36f5935b 2024-11-13T18:37:54,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/78a4556b113a412d8b1611b66a49799a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/78a4556b113a412d8b1611b66a49799a 2024-11-13T18:37:54,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/c1ba8483b5fa418db0e9b057237700ed to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/c1ba8483b5fa418db0e9b057237700ed 2024-11-13T18:37:54,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/bc19d36c8dec4d1fa6a825ce62960284 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/bc19d36c8dec4d1fa6a825ce62960284 2024-11-13T18:37:54,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0fb3caed9277459cb165a7e787fd9bb9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0fb3caed9277459cb165a7e787fd9bb9 2024-11-13T18:37:54,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5f004306f1bd40ba92b6d24b330af1ce to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5f004306f1bd40ba92b6d24b330af1ce 2024-11-13T18:37:54,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e8c8360c0fa0476a9ff08d1940a633b9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e8c8360c0fa0476a9ff08d1940a633b9 2024-11-13T18:37:54,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/208f78b8bda64d719bc5189a04945766 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/208f78b8bda64d719bc5189a04945766 2024-11-13T18:37:54,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7ca4dc7b2ed14009b22f6978d7178830 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7ca4dc7b2ed14009b22f6978d7178830 2024-11-13T18:37:54,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/ae1f5b8f9b3b45779c587a3741fc1e48 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/ae1f5b8f9b3b45779c587a3741fc1e48 2024-11-13T18:37:54,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d62309badfad4c279e31d6791417a18a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d62309badfad4c279e31d6791417a18a 2024-11-13T18:37:54,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/8165115bf660443e9e68bd0ef61cb072 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/8165115bf660443e9e68bd0ef61cb072 2024-11-13T18:37:54,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/757356d034744efb9a802372256a1937 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/757356d034744efb9a802372256a1937 2024-11-13T18:37:54,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/9893e57b63fd4a25a9216135132e275a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/9893e57b63fd4a25a9216135132e275a 2024-11-13T18:37:54,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7f1ad8fa4bcc40f9951f13472057fdb9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/7f1ad8fa4bcc40f9951f13472057fdb9 2024-11-13T18:37:54,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7109e2640e4f4eab276474e4fa0aea to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7109e2640e4f4eab276474e4fa0aea 2024-11-13T18:37:54,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4aac26c722664d088259115c439b72f4 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4aac26c722664d088259115c439b72f4 2024-11-13T18:37:54,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e496171c5d1543419661cd0c352aa1e4 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/e496171c5d1543419661cd0c352aa1e4 2024-11-13T18:37:54,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fd4d493990504072ba3e248dbe530c93 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fd4d493990504072ba3e248dbe530c93 2024-11-13T18:37:54,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a2a4966ef164ed483bf2659b13ee60f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a2a4966ef164ed483bf2659b13ee60f 2024-11-13T18:37:54,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6e5128428d484b969297bd1e8919eab6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6e5128428d484b969297bd1e8919eab6 2024-11-13T18:37:54,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4765b6363fa346789d6ed53ac6611b22 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4765b6363fa346789d6ed53ac6611b22 2024-11-13T18:37:54,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/2b102683097b42beb549c64b036b9d17 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/2b102683097b42beb549c64b036b9d17 2024-11-13T18:37:54,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/da7a4cfeff9b433e970335c3c145aaf8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/da7a4cfeff9b433e970335c3c145aaf8 2024-11-13T18:37:54,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5d5de00f2a474a76b6c1a8c260b01470 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/5d5de00f2a474a76b6c1a8c260b01470 2024-11-13T18:37:54,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a35dc536d4b4a8eb7d683a3dce53579 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0a35dc536d4b4a8eb7d683a3dce53579 2024-11-13T18:37:54,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/a675eace4a834a64bf35b05a7960fbf9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/a675eace4a834a64bf35b05a7960fbf9 2024-11-13T18:37:54,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/3b063fc89f3e40ba91fbae4926120b6e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/3b063fc89f3e40ba91fbae4926120b6e 2024-11-13T18:37:54,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fca36b54dc7f43a4b90327c5cd8a1b28 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/fca36b54dc7f43a4b90327c5cd8a1b28 2024-11-13T18:37:54,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7bc7a189154be1af9c58863f3bcc64 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/6c7bc7a189154be1af9c58863f3bcc64 2024-11-13T18:37:54,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/af5a4c8135f84a32b43067b05f11cfaa to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/af5a4c8135f84a32b43067b05f11cfaa 2024-11-13T18:37:54,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/397753f691f14579865be3581659664b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/397753f691f14579865be3581659664b 2024-11-13T18:37:54,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d7b38e359193458cbaf80e7332530bdc to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/d7b38e359193458cbaf80e7332530bdc 2024-11-13T18:37:54,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0b26755ca20d4921b33c579a7675e90f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/0b26755ca20d4921b33c579a7675e90f 2024-11-13T18:37:54,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4615a8d245504f6da33f82addcf84ce8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/4615a8d245504f6da33f82addcf84ce8 2024-11-13T18:37:54,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1c29ae6b8e734e0badc695d074dfa5a7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1c29ae6b8e734e0badc695d074dfa5a7 2024-11-13T18:37:54,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/31dea4827e7e44e8a596376f3afba8ea to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/31dea4827e7e44e8a596376f3afba8ea 2024-11-13T18:37:54,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1e24591fb97d4cb29131b4b5fba98136 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/1e24591fb97d4cb29131b4b5fba98136 2024-11-13T18:37:54,660 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/recovered.edits/564.seqid, newMaxSeqId=564, maxSeqId=1 2024-11-13T18:37:54,660 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. 2024-11-13T18:37:54,660 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for f8326d4028e020fa48ab6da5484ce85e: Waiting for close lock at 1731523073186Running coprocessor pre-close hooks at 1731523073186Disabling compacts and flushes for region at 1731523073186Disabling writes for close at 1731523073187 (+1 ms)Obtaining lock to block concurrent updates at 1731523073187Preparing flush snapshotting stores in f8326d4028e020fa48ab6da5484ce85e at 1731523073187Finished memstore snapshotting TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731523073187Flushing stores of TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e. at 1731523073188 (+1 ms)Flushing f8326d4028e020fa48ab6da5484ce85e/A: creating writer at 1731523073188Flushing f8326d4028e020fa48ab6da5484ce85e/A: appending metadata at 1731523073191 (+3 ms)Flushing f8326d4028e020fa48ab6da5484ce85e/A: closing flushed file at 1731523073191Flushing f8326d4028e020fa48ab6da5484ce85e/B: creating writer at 1731523073610 (+419 ms)Flushing f8326d4028e020fa48ab6da5484ce85e/B: appending metadata at 1731523073613 (+3 ms)Flushing f8326d4028e020fa48ab6da5484ce85e/B: closing flushed file at 1731523073613Flushing f8326d4028e020fa48ab6da5484ce85e/C: creating writer at 1731523074022 (+409 ms)Flushing f8326d4028e020fa48ab6da5484ce85e/C: appending metadata at 1731523074024 (+2 ms)Flushing f8326d4028e020fa48ab6da5484ce85e/C: closing flushed file at 1731523074024Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55f3baa: reopening flushed file at 1731523074434 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2038636f: reopening flushed file at 1731523074440 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2524c7ae: reopening flushed file at 1731523074446 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for f8326d4028e020fa48ab6da5484ce85e in 1268ms, sequenceid=561, compaction requested=false at 1731523074455 (+9 ms)Writing region close event to WAL at 1731523074655 (+200 ms)Running coprocessor post-close hooks at 1731523074660 (+5 ms)Closed at 1731523074660 2024-11-13T18:37:54,663 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:54,664 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f8326d4028e020fa48ab6da5484ce85e, regionState=CLOSED 2024-11-13T18:37:54,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:37:54,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T18:37:54,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure f8326d4028e020fa48ab6da5484ce85e, server=39e84130bbc9,35235,1731522975587 in 1.6410 sec 2024-11-13T18:37:54,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-13T18:37:54,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8326d4028e020fa48ab6da5484ce85e, UNASSIGN in 1.6530 sec 2024-11-13T18:37:54,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T18:37:54,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6570 sec 2024-11-13T18:37:54,681 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523074681"}]},"ts":"1731523074681"} 2024-11-13T18:37:54,683 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:37:54,683 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:37:54,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6740 sec 2024-11-13T18:37:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T18:37:55,164 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:55,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:37:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:55,166 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:55,167 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T18:37:55,171 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:55,176 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/recovered.edits] 2024-11-13T18:37:55,180 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d29b001784b4eabb671202912116bd4 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/2d29b001784b4eabb671202912116bd4 2024-11-13T18:37:55,181 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f9ca8f5c69ae47d89f02e7788888c5c5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/A/f9ca8f5c69ae47d89f02e7788888c5c5 2024-11-13T18:37:55,183 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0105aae61131487b984755b7e98a433e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0105aae61131487b984755b7e98a433e 2024-11-13T18:37:55,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fed8fc26c4b4fdda1a0254dbada3616 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/B/0fed8fc26c4b4fdda1a0254dbada3616 2024-11-13T18:37:55,187 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/26759e6f1ca84a6a9db72d1c2fb7acfd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/26759e6f1ca84a6a9db72d1c2fb7acfd 2024-11-13T18:37:55,188 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/67e7efe6ce674511b8b414225b7bcb08 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/C/67e7efe6ce674511b8b414225b7bcb08 2024-11-13T18:37:55,192 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/recovered.edits/564.seqid to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e/recovered.edits/564.seqid 2024-11-13T18:37:55,193 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/f8326d4028e020fa48ab6da5484ce85e 2024-11-13T18:37:55,194 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:37:55,196 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:55,201 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:37:55,203 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:37:55,204 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:55,205 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:37:55,206 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731523075206"}]},"ts":"9223372036854775807"} 2024-11-13T18:37:55,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:37:55,208 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f8326d4028e020fa48ab6da5484ce85e, NAME => 'TestAcidGuarantees,,1731523043569.f8326d4028e020fa48ab6da5484ce85e.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:37:55,208 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:37:55,208 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731523075208"}]},"ts":"9223372036854775807"} 2024-11-13T18:37:55,210 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:37:55,211 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:55,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-13T18:37:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T18:37:55,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:37:55,274 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:55,287 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=236 (was 238), OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1804 (was 2705), ProcessCount=11 (was 11), AvailableMemoryMB=8353 (was 3799) - AvailableMemoryMB LEAK? - 2024-11-13T18:37:55,293 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:37:55,298 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=236, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1804, ProcessCount=11, AvailableMemoryMB=8352 2024-11-13T18:37:55,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:37:55,300 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:37:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:55,302 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:37:55,303 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:55,303 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-13T18:37:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T18:37:55,303 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:37:55,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742147_1323 (size=959) 2024-11-13T18:37:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T18:37:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T18:37:55,713 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 2024-11-13T18:37:55,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742148_1324 (size=53) 2024-11-13T18:37:55,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:37:55,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a70474ff27654302c682d5d28687cb3d, disabling compactions & flushes 2024-11-13T18:37:55,723 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:55,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:55,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. after waiting 0 ms 2024-11-13T18:37:55,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:55,723 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:55,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a70474ff27654302c682d5d28687cb3d: Waiting for close lock at 1731523075723Disabling compacts and flushes for region at 1731523075723Disabling writes for close at 1731523075723Writing region close event to WAL at 1731523075723Closed at 1731523075723 2024-11-13T18:37:55,724 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:37:55,725 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731523075724"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731523075724"}]},"ts":"1731523075724"} 2024-11-13T18:37:55,728 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:37:55,733 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:37:55,734 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523075734"}]},"ts":"1731523075734"} 2024-11-13T18:37:55,735 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:37:55,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, ASSIGN}] 2024-11-13T18:37:55,736 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, ASSIGN 2024-11-13T18:37:55,738 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, ASSIGN; state=OFFLINE, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=false 2024-11-13T18:37:55,888 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a70474ff27654302c682d5d28687cb3d, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:55,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, ASSIGN because future has completed 2024-11-13T18:37:55,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:37:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T18:37:56,046 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:56,046 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:37:56,046 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,047 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:37:56,047 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,047 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,048 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,049 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a70474ff27654302c682d5d28687cb3d columnFamilyName A 2024-11-13T18:37:56,050 DEBUG [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:56,050 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(327): Store=a70474ff27654302c682d5d28687cb3d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:56,050 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,051 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a70474ff27654302c682d5d28687cb3d columnFamilyName B 2024-11-13T18:37:56,051 DEBUG [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:56,052 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(327): Store=a70474ff27654302c682d5d28687cb3d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:56,052 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,053 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a70474ff27654302c682d5d28687cb3d columnFamilyName C 2024-11-13T18:37:56,053 DEBUG [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:56,054 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(327): Store=a70474ff27654302c682d5d28687cb3d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:56,054 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,055 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,055 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,056 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,056 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,057 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:37:56,058 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,061 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:37:56,062 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened a70474ff27654302c682d5d28687cb3d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66705151, jitterRate=-0.006015792489051819}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:37:56,062 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:56,062 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for a70474ff27654302c682d5d28687cb3d: Running coprocessor pre-open hook at 1731523076047Writing region info on filesystem at 1731523076047Initializing all the Stores at 1731523076047Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523076048 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523076048Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523076048Cleaning up temporary data from old regions at 1731523076056 (+8 ms)Running coprocessor post-open hooks at 1731523076062 (+6 ms)Region opened successfully at 1731523076062 2024-11-13T18:37:56,063 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., pid=87, masterSystemTime=1731523076043 2024-11-13T18:37:56,065 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:56,065 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:56,065 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a70474ff27654302c682d5d28687cb3d, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:56,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:37:56,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-13T18:37:56,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 in 177 msec 2024-11-13T18:37:56,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-13T18:37:56,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, ASSIGN in 334 msec 2024-11-13T18:37:56,072 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:37:56,072 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523076072"}]},"ts":"1731523076072"} 2024-11-13T18:37:56,074 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:37:56,074 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:37:56,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 774 msec 2024-11-13T18:37:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T18:37:56,444 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:56,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:56,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:56,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:56,451 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:56,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:56,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:56,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b5d6ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:56,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:56,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:56,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:56,454 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3ef9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:56,456 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:56,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:56,459 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:37:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 39e84130bbc9,36105,1731522974847 2024-11-13T18:37:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7684c24b 2024-11-13T18:37:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:37:56,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:37:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:37:56,471 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:37:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:37:56,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742149_1325 (size=995) 2024-11-13T18:37:56,895 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T18:37:56,895 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T18:37:56,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:37:56,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, REOPEN/MOVE}] 2024-11-13T18:37:56,901 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, REOPEN/MOVE 2024-11-13T18:37:56,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a70474ff27654302c682d5d28687cb3d, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:56,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, REOPEN/MOVE because future has completed 2024-11-13T18:37:56,903 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:37:56,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:37:57,055 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,055 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:37:57,055 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing a70474ff27654302c682d5d28687cb3d, disabling compactions & flushes 2024-11-13T18:37:57,055 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,055 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,055 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. after waiting 0 ms 2024-11-13T18:37:57,055 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,067 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:37:57,067 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,067 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for a70474ff27654302c682d5d28687cb3d: Waiting for close lock at 1731523077055Running coprocessor pre-close hooks at 1731523077055Disabling compacts and flushes for region at 1731523077055Disabling writes for close at 1731523077055Writing region close event to WAL at 1731523077056 (+1 ms)Running coprocessor post-close hooks at 1731523077067 (+11 ms)Closed at 1731523077067 2024-11-13T18:37:57,068 WARN [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: a70474ff27654302c682d5d28687cb3d to self. 2024-11-13T18:37:57,071 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,073 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a70474ff27654302c682d5d28687cb3d, regionState=CLOSED 2024-11-13T18:37:57,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:37:57,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-13T18:37:57,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 in 172 msec 2024-11-13T18:37:57,078 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, REOPEN/MOVE; state=CLOSED, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=true 2024-11-13T18:37:57,229 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a70474ff27654302c682d5d28687cb3d, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, REOPEN/MOVE because future has completed 2024-11-13T18:37:57,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:37:57,387 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,387 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:37:57,387 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,387 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:37:57,387 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,388 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,389 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,389 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a70474ff27654302c682d5d28687cb3d columnFamilyName A 2024-11-13T18:37:57,390 DEBUG [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:57,390 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(327): Store=a70474ff27654302c682d5d28687cb3d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:57,391 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,391 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a70474ff27654302c682d5d28687cb3d columnFamilyName B 2024-11-13T18:37:57,391 DEBUG [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:57,392 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(327): Store=a70474ff27654302c682d5d28687cb3d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:57,392 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,392 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a70474ff27654302c682d5d28687cb3d columnFamilyName C 2024-11-13T18:37:57,393 DEBUG [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:57,393 INFO [StoreOpener-a70474ff27654302c682d5d28687cb3d-1 {}] regionserver.HStore(327): Store=a70474ff27654302c682d5d28687cb3d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:37:57,393 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,393 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,394 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,395 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,395 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,395 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:37:57,396 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,397 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened a70474ff27654302c682d5d28687cb3d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67657249, jitterRate=0.008171573281288147}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:37:57,397 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,398 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for a70474ff27654302c682d5d28687cb3d: Running coprocessor pre-open hook at 1731523077388Writing region info on filesystem at 1731523077388Initializing all the Stores at 1731523077388Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523077388Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523077388Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523077388Cleaning up temporary data from old regions at 1731523077395 (+7 ms)Running coprocessor post-open hooks at 1731523077397 (+2 ms)Region opened successfully at 1731523077397 2024-11-13T18:37:57,398 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., pid=92, masterSystemTime=1731523077384 2024-11-13T18:37:57,400 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,400 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,400 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a70474ff27654302c682d5d28687cb3d, regionState=OPEN, openSeqNum=5, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:37:57,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-13T18:37:57,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 in 170 msec 2024-11-13T18:37:57,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-13T18:37:57,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, REOPEN/MOVE in 504 msec 2024-11-13T18:37:57,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-13T18:37:57,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-13T18:37:57,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 936 msec 2024-11-13T18:37:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-13T18:37:57,410 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:57,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18e8d7a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@860604, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff2f4a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f2e380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,417 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e197e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,418 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a76b881, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,420 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194a6f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,421 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dacc64e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,423 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637d2f13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc2055e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3fe857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,426 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cddd12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,428 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cfb5e67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d40eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,431 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38fe7d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,432 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d22b222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,434 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667837ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4279e071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:37:57,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:37:57,437 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:37:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:37:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:37:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a1f89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:37:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:37:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:37:57,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:37:57,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b92ef48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:37:57,439 DEBUG [Thread-1512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,439 DEBUG [Thread-1497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,440 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,440 DEBUG [Thread-1514 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,440 DEBUG [Thread-1508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,440 DEBUG [Thread-1516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,440 DEBUG [Thread-1510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-13T18:37:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,441 DEBUG [Thread-1505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,441 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,441 DEBUG [Thread-1501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,442 DEBUG [Thread-1503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:37:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,442 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:57,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:37:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,443 DEBUG [Thread-1501 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,443 DEBUG [Thread-1501 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:57,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:57,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:37:57,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:37:57,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:37:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:37:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:37:57,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:37:57,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137414aef9ddd440e18e26767f34385e3f_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523077459/Put/seqid=0 2024-11-13T18:37:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742150_1326 (size=17034) 2024-11-13T18:37:57,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:57,509 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137414aef9ddd440e18e26767f34385e3f_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137414aef9ddd440e18e26767f34385e3f_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:57,510 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/5bce60809a7c4685b3340349ed3fb26b, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:37:57,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/5bce60809a7c4685b3340349ed3fb26b is 175, key is test_row_0/A:col10/1731523077459/Put/seqid=0 2024-11-13T18:37:57,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742151_1327 (size=48135) 2024-11-13T18:37:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:37:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523137551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523137555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523137556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523137557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523137569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:37:57,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:37:57,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523137674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523137674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523137674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523137674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523137693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:37:57,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:37:57,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:37:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523137894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:37:57,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:37:57,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:57,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:57,915 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/5bce60809a7c4685b3340349ed3fb26b 2024-11-13T18:37:57,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523137904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523137904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523137913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:57,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523137923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:57,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/db8563b3627044808a0d5b4646b0a0c8 is 50, key is test_row_0/B:col10/1731523077459/Put/seqid=0 2024-11-13T18:37:58,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742152_1328 (size=9657) 2024-11-13T18:37:58,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/db8563b3627044808a0d5b4646b0a0c8 2024-11-13T18:37:58,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/27fd7329e9fc48c7861f10bbfdd349b4 is 50, key is test_row_0/C:col10/1731523077459/Put/seqid=0 2024-11-13T18:37:58,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:37:58,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:58,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:37:58,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:58,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:58,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:37:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:37:58,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742153_1329 (size=9657) 2024-11-13T18:37:58,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/27fd7329e9fc48c7861f10bbfdd349b4 2024-11-13T18:37:58,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/5bce60809a7c4685b3340349ed3fb26b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/5bce60809a7c4685b3340349ed3fb26b 2024-11-13T18:37:58,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/5bce60809a7c4685b3340349ed3fb26b, entries=250, sequenceid=17, filesize=47.0 K 2024-11-13T18:37:58,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/db8563b3627044808a0d5b4646b0a0c8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/db8563b3627044808a0d5b4646b0a0c8 2024-11-13T18:37:58,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/db8563b3627044808a0d5b4646b0a0c8, entries=100, sequenceid=17, filesize=9.4 K 2024-11-13T18:37:58,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/27fd7329e9fc48c7861f10bbfdd349b4 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/27fd7329e9fc48c7861f10bbfdd349b4 2024-11-13T18:37:58,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/27fd7329e9fc48c7861f10bbfdd349b4, entries=100, sequenceid=17, filesize=9.4 K 2024-11-13T18:37:58,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a70474ff27654302c682d5d28687cb3d in 681ms, sequenceid=17, compaction requested=false 2024-11-13T18:37:58,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:37:58,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T18:37:58,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:58,218 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T18:37:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:58,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:37:58,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111302478100beb6493686888e32582f3167_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523078217/Put/seqid=0 2024-11-13T18:37:58,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742154_1330 (size=14594) 2024-11-13T18:37:58,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:37:58,243 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111302478100beb6493686888e32582f3167_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302478100beb6493686888e32582f3167_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:58,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/e13c21668e20451190ea0431e1bc739b, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:37:58,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/e13c21668e20451190ea0431e1bc739b is 175, key is test_row_0/A:col10/1731523078217/Put/seqid=0 2024-11-13T18:37:58,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523138243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523138247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742155_1331 (size=39549) 2024-11-13T18:37:58,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523138248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523138249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523138250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523138353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523138363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523138373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523138373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523138563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:37:58,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523138573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523138583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523138593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,656 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/e13c21668e20451190ea0431e1bc739b 2024-11-13T18:37:58,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/2c9291caf52b49cea3bca210d190920a is 50, key is test_row_0/B:col10/1731523078217/Put/seqid=0 2024-11-13T18:37:58,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742156_1332 (size=12001) 2024-11-13T18:37:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523138764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523138873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523138893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523138903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523138903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:59,068 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/2c9291caf52b49cea3bca210d190920a 2024-11-13T18:37:59,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/cd9f612650a64a4abd8bb9802b023966 is 50, key is test_row_0/C:col10/1731523078217/Put/seqid=0 2024-11-13T18:37:59,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742157_1333 (size=12001) 2024-11-13T18:37:59,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523139384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:59,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:59,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523139413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523139423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523139423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:37:59,487 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/cd9f612650a64a4abd8bb9802b023966 2024-11-13T18:37:59,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/e13c21668e20451190ea0431e1bc739b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e13c21668e20451190ea0431e1bc739b 2024-11-13T18:37:59,495 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e13c21668e20451190ea0431e1bc739b, entries=200, sequenceid=41, filesize=38.6 K 2024-11-13T18:37:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/2c9291caf52b49cea3bca210d190920a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2c9291caf52b49cea3bca210d190920a 2024-11-13T18:37:59,501 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2c9291caf52b49cea3bca210d190920a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-13T18:37:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/cd9f612650a64a4abd8bb9802b023966 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/cd9f612650a64a4abd8bb9802b023966 2024-11-13T18:37:59,507 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/cd9f612650a64a4abd8bb9802b023966, entries=150, sequenceid=41, filesize=11.7 K 2024-11-13T18:37:59,508 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a70474ff27654302c682d5d28687cb3d in 1290ms, sequenceid=41, compaction requested=false 2024-11-13T18:37:59,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:37:59,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:59,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-13T18:37:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-13T18:37:59,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-13T18:37:59,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0670 sec 2024-11-13T18:37:59,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.0720 sec 2024-11-13T18:37:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T18:37:59,594 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:37:59,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:37:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-13T18:37:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:37:59,596 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:37:59,596 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:37:59,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:37:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:37:59,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T18:37:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:37:59,749 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T18:37:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bd13eb34b390483191dfb97d24a67cb7_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523078246/Put/seqid=0 2024-11-13T18:37:59,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742158_1334 (size=12154) 2024-11-13T18:37:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:37:59,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:37:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:37:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:37:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523139964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:37:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:37:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:00,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523140083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:00,177 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bd13eb34b390483191dfb97d24a67cb7_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd13eb34b390483191dfb97d24a67cb7_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/bea4122a257545d3bc238404d89c35fa, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/bea4122a257545d3bc238404d89c35fa is 175, key is test_row_0/A:col10/1731523078246/Put/seqid=0 2024-11-13T18:38:00,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742159_1335 (size=30955) 2024-11-13T18:38:00,191 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:38:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:38:00,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523140293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:00,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523140393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:00,402 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523140433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:00,445 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523140443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523140443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:00,453 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:00,454 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_1' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:00,584 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/bea4122a257545d3bc238404d89c35fa 2024-11-13T18:38:00,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/69e1107c71e349fca397ecb82fa7b7bd is 50, key is test_row_0/B:col10/1731523078246/Put/seqid=0 2024-11-13T18:38:00,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742160_1336 (size=12001) 2024-11-13T18:38:00,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523140613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:38:00,998 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/69e1107c71e349fca397ecb82fa7b7bd 2024-11-13T18:38:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/873c81e70afc4c09a91034f28f9fa6a1 is 50, key is test_row_0/C:col10/1731523078246/Put/seqid=0 2024-11-13T18:38:01,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742161_1337 (size=12001) 2024-11-13T18:38:01,015 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/873c81e70afc4c09a91034f28f9fa6a1 2024-11-13T18:38:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/bea4122a257545d3bc238404d89c35fa as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/bea4122a257545d3bc238404d89c35fa 2024-11-13T18:38:01,023 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/bea4122a257545d3bc238404d89c35fa, entries=150, sequenceid=54, filesize=30.2 K 2024-11-13T18:38:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/69e1107c71e349fca397ecb82fa7b7bd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69e1107c71e349fca397ecb82fa7b7bd 2024-11-13T18:38:01,026 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69e1107c71e349fca397ecb82fa7b7bd, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T18:38:01,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/873c81e70afc4c09a91034f28f9fa6a1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/873c81e70afc4c09a91034f28f9fa6a1 2024-11-13T18:38:01,030 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/873c81e70afc4c09a91034f28f9fa6a1, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T18:38:01,031 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a70474ff27654302c682d5d28687cb3d in 1282ms, sequenceid=54, compaction requested=true 2024-11-13T18:38:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-13T18:38:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-13T18:38:01,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T18:38:01,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-13T18:38:01,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.4410 sec 2024-11-13T18:38:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:01,132 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:01,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T18:38:01,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d17cf084152f4259857f0d60c9e910c0_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523079955/Put/seqid=0 2024-11-13T18:38:01,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742162_1338 (size=14594) 2024-11-13T18:38:01,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523141214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:01,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523141323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:01,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:01,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d17cf084152f4259857f0d60c9e910c0_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d17cf084152f4259857f0d60c9e910c0_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:01,554 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/6be19aa8d16a47c0ab4ea70f4afe71be, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:01,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/6be19aa8d16a47c0ab4ea70f4afe71be is 175, key is test_row_0/A:col10/1731523079955/Put/seqid=0 2024-11-13T18:38:01,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523141543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:01,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742163_1339 (size=39549) 2024-11-13T18:38:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T18:38:01,763 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:01,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-13T18:38:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:38:01,766 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:01,766 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:01,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:38:01,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523141864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:01,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:01,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:01,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:01,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:01,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:01,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:01,960 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/6be19aa8d16a47c0ab4ea70f4afe71be 2024-11-13T18:38:01,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/6908ee6897f74e3d86c7b8a66c932109 is 50, key is test_row_0/B:col10/1731523079955/Put/seqid=0 2024-11-13T18:38:01,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742164_1340 (size=12001) 2024-11-13T18:38:02,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:38:02,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/6908ee6897f74e3d86c7b8a66c932109 2024-11-13T18:38:02,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e909a477a5654687bab3683429c8c8b0 is 50, key is test_row_0/C:col10/1731523079955/Put/seqid=0 2024-11-13T18:38:02,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523142383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742165_1341 (size=12001) 2024-11-13T18:38:02,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e909a477a5654687bab3683429c8c8b0 2024-11-13T18:38:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:38:02,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/6be19aa8d16a47c0ab4ea70f4afe71be as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6be19aa8d16a47c0ab4ea70f4afe71be 2024-11-13T18:38:02,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6be19aa8d16a47c0ab4ea70f4afe71be, entries=200, sequenceid=78, filesize=38.6 K 2024-11-13T18:38:02,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/6908ee6897f74e3d86c7b8a66c932109 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6908ee6897f74e3d86c7b8a66c932109 2024-11-13T18:38:02,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6908ee6897f74e3d86c7b8a66c932109, entries=150, sequenceid=78, filesize=11.7 K 2024-11-13T18:38:02,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e909a477a5654687bab3683429c8c8b0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e909a477a5654687bab3683429c8c8b0 2024-11-13T18:38:02,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e909a477a5654687bab3683429c8c8b0, entries=150, sequenceid=78, filesize=11.7 K 2024-11-13T18:38:02,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a70474ff27654302c682d5d28687cb3d in 1274ms, sequenceid=78, compaction requested=true 2024-11-13T18:38:02,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:02,407 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:02,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:02,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:02,407 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:02,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:02,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:02,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:02,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:02,409 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:02,409 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/A is initiating minor compaction (all files) 2024-11-13T18:38:02,412 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/A in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,412 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/5bce60809a7c4685b3340349ed3fb26b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e13c21668e20451190ea0431e1bc739b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/bea4122a257545d3bc238404d89c35fa, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6be19aa8d16a47c0ab4ea70f4afe71be] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=154.5 K 2024-11-13T18:38:02,412 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,412 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/5bce60809a7c4685b3340349ed3fb26b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e13c21668e20451190ea0431e1bc739b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/bea4122a257545d3bc238404d89c35fa, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6be19aa8d16a47c0ab4ea70f4afe71be] 2024-11-13T18:38:02,413 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bce60809a7c4685b3340349ed3fb26b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731523077450 2024-11-13T18:38:02,413 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e13c21668e20451190ea0431e1bc739b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731523077505 2024-11-13T18:38:02,414 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:02,414 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea4122a257545d3bc238404d89c35fa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523078235 2024-11-13T18:38:02,414 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/B is initiating minor compaction (all files) 2024-11-13T18:38:02,414 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/B in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,414 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/db8563b3627044808a0d5b4646b0a0c8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2c9291caf52b49cea3bca210d190920a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69e1107c71e349fca397ecb82fa7b7bd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6908ee6897f74e3d86c7b8a66c932109] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=44.6 K 2024-11-13T18:38:02,414 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be19aa8d16a47c0ab4ea70f4afe71be, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523079925 2024-11-13T18:38:02,414 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting db8563b3627044808a0d5b4646b0a0c8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731523077459 2024-11-13T18:38:02,415 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9291caf52b49cea3bca210d190920a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731523077505 2024-11-13T18:38:02,415 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 69e1107c71e349fca397ecb82fa7b7bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523078235 2024-11-13T18:38:02,415 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6908ee6897f74e3d86c7b8a66c932109, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523079955 2024-11-13T18:38:02,423 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:02,423 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:02,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T18:38:02,430 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113e8ac1a12c2d94d51a14ca876d11e9db5_a70474ff27654302c682d5d28687cb3d store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:02,432 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#B#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:02,432 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113e8ac1a12c2d94d51a14ca876d11e9db5_a70474ff27654302c682d5d28687cb3d, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:02,432 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/00f55f51bfc14f5899dab7df9f6d1daf is 50, key is test_row_0/B:col10/1731523079955/Put/seqid=0 2024-11-13T18:38:02,432 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e8ac1a12c2d94d51a14ca876d11e9db5_a70474ff27654302c682d5d28687cb3d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:02,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113713084b215464812bd5bb6a20b627f08_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523081212/Put/seqid=0 2024-11-13T18:38:02,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742166_1342 (size=12139) 2024-11-13T18:38:02,453 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/00f55f51bfc14f5899dab7df9f6d1daf as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/00f55f51bfc14f5899dab7df9f6d1daf 2024-11-13T18:38:02,457 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a70474ff27654302c682d5d28687cb3d/B of a70474ff27654302c682d5d28687cb3d into 00f55f51bfc14f5899dab7df9f6d1daf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:02,458 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:02,458 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/B, priority=12, startTime=1731523082407; duration=0sec 2024-11-13T18:38:02,458 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:02,458 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:02,458 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:02,459 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:02,459 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/C is initiating minor compaction (all files) 2024-11-13T18:38:02,459 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/C in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,459 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/27fd7329e9fc48c7861f10bbfdd349b4, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/cd9f612650a64a4abd8bb9802b023966, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/873c81e70afc4c09a91034f28f9fa6a1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e909a477a5654687bab3683429c8c8b0] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=44.6 K 2024-11-13T18:38:02,460 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fd7329e9fc48c7861f10bbfdd349b4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731523077459 2024-11-13T18:38:02,460 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9f612650a64a4abd8bb9802b023966, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731523077505 2024-11-13T18:38:02,461 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 873c81e70afc4c09a91034f28f9fa6a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523078235 2024-11-13T18:38:02,461 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e909a477a5654687bab3683429c8c8b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523079955 2024-11-13T18:38:02,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742167_1343 (size=4469) 2024-11-13T18:38:02,470 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#A#compaction#294 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742168_1344 (size=14594) 2024-11-13T18:38:02,471 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/decd69a2bebd4a74b50a92a1df6543fa is 175, key is test_row_0/A:col10/1731523079955/Put/seqid=0 2024-11-13T18:38:02,475 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:02,476 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/d84003405f4c4c6a9ef210e01bdd4058 is 50, key is test_row_0/C:col10/1731523079955/Put/seqid=0 2024-11-13T18:38:02,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742169_1345 (size=31093) 2024-11-13T18:38:02,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742170_1346 (size=12139) 2024-11-13T18:38:02,489 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/d84003405f4c4c6a9ef210e01bdd4058 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/d84003405f4c4c6a9ef210e01bdd4058 2024-11-13T18:38:02,494 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a70474ff27654302c682d5d28687cb3d/C of a70474ff27654302c682d5d28687cb3d into d84003405f4c4c6a9ef210e01bdd4058(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:02,494 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:02,494 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/C, priority=12, startTime=1731523082407; duration=0sec 2024-11-13T18:38:02,494 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:02,494 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:02,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523142526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523142539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523142541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523142541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523142643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523142663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523142663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523142664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:02,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:02,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:02,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523142863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,875 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113713084b215464812bd5bb6a20b627f08_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113713084b215464812bd5bb6a20b627f08_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:02,876 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8f00a24dd2f74b52aea1e062a7312f83, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:02,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8f00a24dd2f74b52aea1e062a7312f83 is 175, key is test_row_0/A:col10/1731523081212/Put/seqid=0 2024-11-13T18:38:02,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742171_1347 (size=39549) 2024-11-13T18:38:02,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523142884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523142884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523142884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:02,890 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/decd69a2bebd4a74b50a92a1df6543fa as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/decd69a2bebd4a74b50a92a1df6543fa 2024-11-13T18:38:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:02,894 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a70474ff27654302c682d5d28687cb3d/A of a70474ff27654302c682d5d28687cb3d into decd69a2bebd4a74b50a92a1df6543fa(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:02,894 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:02,894 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/A, priority=12, startTime=1731523082407; duration=0sec 2024-11-13T18:38:02,894 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:02,894 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:38:02,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:02,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523143183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523143193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523143193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523143193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,281 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8f00a24dd2f74b52aea1e062a7312f83 2024-11-13T18:38:03,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/ef13cfec09824fceb8eeb772bfca94bc is 50, key is test_row_0/B:col10/1731523081212/Put/seqid=0 2024-11-13T18:38:03,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742172_1348 (size=12001) 2024-11-13T18:38:03,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523143393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,396 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:03,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/ef13cfec09824fceb8eeb772bfca94bc 2024-11-13T18:38:03,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/dc71af65307041d7bda63e35f24e1d7f is 50, key is test_row_0/C:col10/1731523081212/Put/seqid=0 2024-11-13T18:38:03,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742173_1349 (size=12001) 2024-11-13T18:38:03,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/dc71af65307041d7bda63e35f24e1d7f 2024-11-13T18:38:03,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8f00a24dd2f74b52aea1e062a7312f83 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8f00a24dd2f74b52aea1e062a7312f83 2024-11-13T18:38:03,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8f00a24dd2f74b52aea1e062a7312f83, entries=200, sequenceid=91, filesize=38.6 K 2024-11-13T18:38:03,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/ef13cfec09824fceb8eeb772bfca94bc as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/ef13cfec09824fceb8eeb772bfca94bc 2024-11-13T18:38:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523143704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523143704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523143704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/ef13cfec09824fceb8eeb772bfca94bc, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T18:38:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/dc71af65307041d7bda63e35f24e1d7f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/dc71af65307041d7bda63e35f24e1d7f 2024-11-13T18:38:03,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/dc71af65307041d7bda63e35f24e1d7f, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T18:38:03,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for a70474ff27654302c682d5d28687cb3d in 1298ms, sequenceid=91, compaction requested=false 2024-11-13T18:38:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:03,725 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:03,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:38:03,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c9d11d09f8124c82ad4ad5f678c3d0cd_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523082539/Put/seqid=0 2024-11-13T18:38:03,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742174_1350 (size=14594) 2024-11-13T18:38:03,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523143809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:38:03,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:03,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:03,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:03,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:03,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523143923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:04,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:04,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:04,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:04,156 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523144145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:04,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c9d11d09f8124c82ad4ad5f678c3d0cd_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c9d11d09f8124c82ad4ad5f678c3d0cd_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:04,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/2bc03960a6824fda8d5d1bc1189d1795, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:04,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/2bc03960a6824fda8d5d1bc1189d1795 is 175, key is test_row_0/A:col10/1731523082539/Put/seqid=0 2024-11-13T18:38:04,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742175_1351 (size=39549) 2024-11-13T18:38:04,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/2bc03960a6824fda8d5d1bc1189d1795 2024-11-13T18:38:04,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/e0ad8027ef09453a804f363bb69ff8c9 is 50, key is test_row_0/B:col10/1731523082539/Put/seqid=0 2024-11-13T18:38:04,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742176_1352 (size=12001) 2024-11-13T18:38:04,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523144463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:04,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:04,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:04,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/e0ad8027ef09453a804f363bb69ff8c9 2024-11-13T18:38:04,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/14049248143d48c19d174a4eac6c799c is 50, key is test_row_0/C:col10/1731523082539/Put/seqid=0 2024-11-13T18:38:04,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742177_1353 (size=12001) 2024-11-13T18:38:04,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/14049248143d48c19d174a4eac6c799c 2024-11-13T18:38:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/2bc03960a6824fda8d5d1bc1189d1795 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2bc03960a6824fda8d5d1bc1189d1795 2024-11-13T18:38:04,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2bc03960a6824fda8d5d1bc1189d1795, entries=200, sequenceid=119, filesize=38.6 K 2024-11-13T18:38:04,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/e0ad8027ef09453a804f363bb69ff8c9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/e0ad8027ef09453a804f363bb69ff8c9 2024-11-13T18:38:04,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/e0ad8027ef09453a804f363bb69ff8c9, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T18:38:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/14049248143d48c19d174a4eac6c799c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/14049248143d48c19d174a4eac6c799c 2024-11-13T18:38:04,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/14049248143d48c19d174a4eac6c799c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T18:38:04,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a70474ff27654302c682d5d28687cb3d in 1007ms, sequenceid=119, compaction requested=true 2024-11-13T18:38:04,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:04,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:04,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:04,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:04,733 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:04,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:04,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:04,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:38:04,733 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:04,734 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:04,735 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/A is initiating minor compaction (all files) 2024-11-13T18:38:04,735 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/A in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,735 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/decd69a2bebd4a74b50a92a1df6543fa, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8f00a24dd2f74b52aea1e062a7312f83, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2bc03960a6824fda8d5d1bc1189d1795] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=107.6 K 2024-11-13T18:38:04,735 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,735 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/decd69a2bebd4a74b50a92a1df6543fa, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8f00a24dd2f74b52aea1e062a7312f83, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2bc03960a6824fda8d5d1bc1189d1795] 2024-11-13T18:38:04,735 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:04,736 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/B is initiating minor compaction (all files) 2024-11-13T18:38:04,736 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/B in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,736 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/00f55f51bfc14f5899dab7df9f6d1daf, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/ef13cfec09824fceb8eeb772bfca94bc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/e0ad8027ef09453a804f363bb69ff8c9] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=35.3 K 2024-11-13T18:38:04,736 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting decd69a2bebd4a74b50a92a1df6543fa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523079955 2024-11-13T18:38:04,737 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f55f51bfc14f5899dab7df9f6d1daf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523079955 2024-11-13T18:38:04,737 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f00a24dd2f74b52aea1e062a7312f83, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731523081175 2024-11-13T18:38:04,737 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc03960a6824fda8d5d1bc1189d1795, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731523082521 2024-11-13T18:38:04,737 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef13cfec09824fceb8eeb772bfca94bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731523081175 2024-11-13T18:38:04,738 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0ad8027ef09453a804f363bb69ff8c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731523082539 2024-11-13T18:38:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:04,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:04,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:38:04,756 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:04,757 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/3ae06d958be946129b88ffdba7b7a9b8 is 50, key is test_row_0/B:col10/1731523082539/Put/seqid=0 2024-11-13T18:38:04,780 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:04,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bf170dce92a24aeba4dba1208587b922_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523083807/Put/seqid=0 2024-11-13T18:38:04,796 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411133db50dae11054ee683efaa6605fc967e_a70474ff27654302c682d5d28687cb3d store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:04,798 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411133db50dae11054ee683efaa6605fc967e_a70474ff27654302c682d5d28687cb3d, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:04,798 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133db50dae11054ee683efaa6605fc967e_a70474ff27654302c682d5d28687cb3d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742178_1354 (size=12241) 2024-11-13T18:38:04,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742179_1355 (size=12254) 2024-11-13T18:38:04,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:04,828 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bf170dce92a24aeba4dba1208587b922_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bf170dce92a24aeba4dba1208587b922_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:04,829 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/d079af0742f1424a8702ec0b50d5a58a, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:04,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/d079af0742f1424a8702ec0b50d5a58a is 175, key is test_row_0/A:col10/1731523083807/Put/seqid=0 2024-11-13T18:38:04,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742180_1356 (size=4469) 2024-11-13T18:38:04,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,851 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#A#compaction#304 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:04,852 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8a1c9d55e2ff4ccb9d2464ac22bc1032 is 175, key is test_row_0/A:col10/1731523082539/Put/seqid=0 2024-11-13T18:38:04,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742181_1357 (size=31055) 2024-11-13T18:38:04,863 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/d079af0742f1424a8702ec0b50d5a58a 2024-11-13T18:38:04,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742182_1358 (size=31195) 2024-11-13T18:38:04,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:04,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523144856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:04,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523144863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:04,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/9e9841dca1f84364902e65c9e7fff0a9 is 50, key is test_row_0/B:col10/1731523083807/Put/seqid=0 2024-11-13T18:38:04,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742183_1359 (size=12101) 2024-11-13T18:38:04,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523144881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:04,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:04,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523144983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523144983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523144993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523145004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:05,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:05,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523145203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,218 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/3ae06d958be946129b88ffdba7b7a9b8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ae06d958be946129b88ffdba7b7a9b8 2024-11-13T18:38:05,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523145213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,227 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/B of a70474ff27654302c682d5d28687cb3d into 3ae06d958be946129b88ffdba7b7a9b8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,227 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,227 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/B, priority=13, startTime=1731523084732; duration=0sec 2024-11-13T18:38:05,227 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:05,227 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:05,227 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:05,228 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:05,228 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/C is initiating minor compaction (all files) 2024-11-13T18:38:05,228 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/C in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,228 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/d84003405f4c4c6a9ef210e01bdd4058, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/dc71af65307041d7bda63e35f24e1d7f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/14049248143d48c19d174a4eac6c799c] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=35.3 K 2024-11-13T18:38:05,228 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d84003405f4c4c6a9ef210e01bdd4058, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523079955 2024-11-13T18:38:05,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523145223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,229 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc71af65307041d7bda63e35f24e1d7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731523081175 2024-11-13T18:38:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,229 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14049248143d48c19d174a4eac6c799c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731523082539 2024-11-13T18:38:05,236 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:05,236 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/3360c7e3caba4fa6957e281382a85758 is 50, key is test_row_0/C:col10/1731523082539/Put/seqid=0 2024-11-13T18:38:05,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742184_1360 (size=12241) 2024-11-13T18:38:05,286 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8a1c9d55e2ff4ccb9d2464ac22bc1032 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8a1c9d55e2ff4ccb9d2464ac22bc1032 2024-11-13T18:38:05,293 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T18:38:05,293 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T18:38:05,294 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/A of a70474ff27654302c682d5d28687cb3d into 8a1c9d55e2ff4ccb9d2464ac22bc1032(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:05,294 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:05,294 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/A, priority=13, startTime=1731523084732; duration=0sec 2024-11-13T18:38:05,294 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:05,294 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:05,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/9e9841dca1f84364902e65c9e7fff0a9 2024-11-13T18:38:05,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/ad97c97ec12c4e0a8f03de0f3c18e28c is 50, key is test_row_0/C:col10/1731523083807/Put/seqid=0 2024-11-13T18:38:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742185_1361 (size=12101) 2024-11-13T18:38:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523145423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,438 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523145523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523145533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523145533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:05,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:05,647 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/3360c7e3caba4fa6957e281382a85758 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/3360c7e3caba4fa6957e281382a85758 2024-11-13T18:38:05,651 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/C of a70474ff27654302c682d5d28687cb3d into 3360c7e3caba4fa6957e281382a85758(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:05,651 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:05,651 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/C, priority=13, startTime=1731523084733; duration=0sec 2024-11-13T18:38:05,652 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:05,652 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:05,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/ad97c97ec12c4e0a8f03de0f3c18e28c 2024-11-13T18:38:05,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/d079af0742f1424a8702ec0b50d5a58a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/d079af0742f1424a8702ec0b50d5a58a 2024-11-13T18:38:05,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/d079af0742f1424a8702ec0b50d5a58a, entries=150, sequenceid=132, filesize=30.3 K 2024-11-13T18:38:05,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/9e9841dca1f84364902e65c9e7fff0a9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/9e9841dca1f84364902e65c9e7fff0a9 2024-11-13T18:38:05,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/9e9841dca1f84364902e65c9e7fff0a9, entries=150, sequenceid=132, filesize=11.8 K 2024-11-13T18:38:05,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/ad97c97ec12c4e0a8f03de0f3c18e28c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad97c97ec12c4e0a8f03de0f3c18e28c 2024-11-13T18:38:05,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad97c97ec12c4e0a8f03de0f3c18e28c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-13T18:38:05,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a70474ff27654302c682d5d28687cb3d in 997ms, sequenceid=132, compaction requested=false 2024-11-13T18:38:05,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:05,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T18:38:05,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,761 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T18:38:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135e9ed44a1ba6484c9e5f910f0215e95f_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523084877/Put/seqid=0 2024-11-13T18:38:05,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742186_1362 (size=12304) 2024-11-13T18:38:05,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,787 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135e9ed44a1ba6484c9e5f910f0215e95f_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135e9ed44a1ba6484c9e5f910f0215e95f_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:05,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/14763a04eec2421ab9c8946ce5c07846, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/14763a04eec2421ab9c8946ce5c07846 is 175, key is test_row_0/A:col10/1731523084877/Put/seqid=0 2024-11-13T18:38:05,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742187_1363 (size=31105) 2024-11-13T18:38:05,800 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/14763a04eec2421ab9c8946ce5c07846 2024-11-13T18:38:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/1a82204a75af44b69e849c815969987d is 50, key is test_row_0/B:col10/1731523084877/Put/seqid=0 2024-11-13T18:38:05,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742188_1364 (size=12151) 2024-11-13T18:38:05,846 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/1a82204a75af44b69e849c815969987d 2024-11-13T18:38:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e4e33fecb57e4377819d3dc057e774c0 is 50, key is test_row_0/C:col10/1731523084877/Put/seqid=0 2024-11-13T18:38:05,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742189_1365 (size=12151) 2024-11-13T18:38:05,885 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e4e33fecb57e4377819d3dc057e774c0 2024-11-13T18:38:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/14763a04eec2421ab9c8946ce5c07846 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/14763a04eec2421ab9c8946ce5c07846 2024-11-13T18:38:05,911 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/14763a04eec2421ab9c8946ce5c07846, entries=150, sequenceid=158, filesize=30.4 K 2024-11-13T18:38:05,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/1a82204a75af44b69e849c815969987d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1a82204a75af44b69e849c815969987d 2024-11-13T18:38:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:38:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,931 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1a82204a75af44b69e849c815969987d, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T18:38:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e4e33fecb57e4377819d3dc057e774c0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e4e33fecb57e4377819d3dc057e774c0 2024-11-13T18:38:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,937 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e4e33fecb57e4377819d3dc057e774c0, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T18:38:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,938 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for a70474ff27654302c682d5d28687cb3d in 177ms, sequenceid=158, compaction requested=true 2024-11-13T18:38:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-13T18:38:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-13T18:38:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-13T18:38:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1740 sec 2024-11-13T18:38:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 4.1790 sec 2024-11-13T18:38:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:06,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:38:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113967c869684bc46349ac46238758d8bc2_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523086183/Put/seqid=0 2024-11-13T18:38:06,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742190_1366 (size=19774) 2024-11-13T18:38:06,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523146448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523146448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523146449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523146451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523146573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523146573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523146573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523146573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523146793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523146793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523146793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523146793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:06,823 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:06,827 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113967c869684bc46349ac46238758d8bc2_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113967c869684bc46349ac46238758d8bc2_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:06,829 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/fab4f71d52204f8f991f48ac2f55cce8, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:06,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/fab4f71d52204f8f991f48ac2f55cce8 is 175, key is test_row_0/A:col10/1731523086183/Put/seqid=0 2024-11-13T18:38:06,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742191_1367 (size=57033) 2024-11-13T18:38:06,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/fab4f71d52204f8f991f48ac2f55cce8 2024-11-13T18:38:06,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/6ef52441cf884d4489c065ccf56544a9 is 50, key is test_row_0/B:col10/1731523086183/Put/seqid=0 2024-11-13T18:38:06,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742192_1368 (size=12151) 2024-11-13T18:38:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523147113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523147114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523147114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523147114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/6ef52441cf884d4489c065ccf56544a9 2024-11-13T18:38:07,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e2ae85213a9b47e18348eca893a04ee1 is 50, key is test_row_0/C:col10/1731523086183/Put/seqid=0 2024-11-13T18:38:07,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742193_1369 (size=12151) 2024-11-13T18:38:07,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e2ae85213a9b47e18348eca893a04ee1 2024-11-13T18:38:07,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/fab4f71d52204f8f991f48ac2f55cce8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/fab4f71d52204f8f991f48ac2f55cce8 2024-11-13T18:38:07,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/fab4f71d52204f8f991f48ac2f55cce8, entries=300, sequenceid=169, filesize=55.7 K 2024-11-13T18:38:07,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/6ef52441cf884d4489c065ccf56544a9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6ef52441cf884d4489c065ccf56544a9 2024-11-13T18:38:07,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6ef52441cf884d4489c065ccf56544a9, entries=150, sequenceid=169, filesize=11.9 K 2024-11-13T18:38:07,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e2ae85213a9b47e18348eca893a04ee1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e2ae85213a9b47e18348eca893a04ee1 2024-11-13T18:38:07,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e2ae85213a9b47e18348eca893a04ee1, entries=150, sequenceid=169, filesize=11.9 K 2024-11-13T18:38:07,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a70474ff27654302c682d5d28687cb3d in 1086ms, sequenceid=169, compaction requested=true 2024-11-13T18:38:07,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:07,408 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:07,408 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:07,410 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150388 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:07,410 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/A is initiating minor compaction (all files) 2024-11-13T18:38:07,411 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/A in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:07,411 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8a1c9d55e2ff4ccb9d2464ac22bc1032, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/d079af0742f1424a8702ec0b50d5a58a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/14763a04eec2421ab9c8946ce5c07846, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/fab4f71d52204f8f991f48ac2f55cce8] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=146.9 K 2024-11-13T18:38:07,411 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:07,411 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8a1c9d55e2ff4ccb9d2464ac22bc1032, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/d079af0742f1424a8702ec0b50d5a58a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/14763a04eec2421ab9c8946ce5c07846, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/fab4f71d52204f8f991f48ac2f55cce8] 2024-11-13T18:38:07,413 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:07,413 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/B is initiating minor compaction (all files) 2024-11-13T18:38:07,413 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/B in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:07,413 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ae06d958be946129b88ffdba7b7a9b8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/9e9841dca1f84364902e65c9e7fff0a9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1a82204a75af44b69e849c815969987d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6ef52441cf884d4489c065ccf56544a9] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=47.5 K 2024-11-13T18:38:07,413 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a1c9d55e2ff4ccb9d2464ac22bc1032, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731523082539 2024-11-13T18:38:07,413 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae06d958be946129b88ffdba7b7a9b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731523082539 2024-11-13T18:38:07,414 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9841dca1f84364902e65c9e7fff0a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731523083734 2024-11-13T18:38:07,414 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d079af0742f1424a8702ec0b50d5a58a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731523083734 2024-11-13T18:38:07,415 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a82204a75af44b69e849c815969987d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731523084833 2024-11-13T18:38:07,415 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14763a04eec2421ab9c8946ce5c07846, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731523084833 2024-11-13T18:38:07,415 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fab4f71d52204f8f991f48ac2f55cce8, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523086181 2024-11-13T18:38:07,415 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef52441cf884d4489c065ccf56544a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523086183 2024-11-13T18:38:07,440 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#B#compaction#315 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:07,440 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/98188398ef804fecb435417e80c5dd32 is 50, key is test_row_0/B:col10/1731523086183/Put/seqid=0 2024-11-13T18:38:07,446 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:07,473 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111341fda89eb66249be9ffc7342911abbb7_a70474ff27654302c682d5d28687cb3d store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:07,475 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111341fda89eb66249be9ffc7342911abbb7_a70474ff27654302c682d5d28687cb3d, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:07,476 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111341fda89eb66249be9ffc7342911abbb7_a70474ff27654302c682d5d28687cb3d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:07,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742194_1370 (size=12527) 2024-11-13T18:38:07,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742195_1371 (size=4469) 2024-11-13T18:38:07,505 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/98188398ef804fecb435417e80c5dd32 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/98188398ef804fecb435417e80c5dd32 2024-11-13T18:38:07,512 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a70474ff27654302c682d5d28687cb3d/B of a70474ff27654302c682d5d28687cb3d into 98188398ef804fecb435417e80c5dd32(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:07,512 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:07,512 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/B, priority=12, startTime=1731523087408; duration=0sec 2024-11-13T18:38:07,512 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:07,512 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:07,512 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:07,513 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:07,513 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/C is initiating minor compaction (all files) 2024-11-13T18:38:07,513 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/C in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:07,513 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/3360c7e3caba4fa6957e281382a85758, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad97c97ec12c4e0a8f03de0f3c18e28c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e4e33fecb57e4377819d3dc057e774c0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e2ae85213a9b47e18348eca893a04ee1] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=47.5 K 2024-11-13T18:38:07,514 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3360c7e3caba4fa6957e281382a85758, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731523082539 2024-11-13T18:38:07,514 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ad97c97ec12c4e0a8f03de0f3c18e28c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731523083734 2024-11-13T18:38:07,515 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e4e33fecb57e4377819d3dc057e774c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731523084833 2024-11-13T18:38:07,515 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ae85213a9b47e18348eca893a04ee1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523086183 2024-11-13T18:38:07,540 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#C#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:07,540 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/da850c7774954c5ab7ca7ee8d87ec9e6 is 50, key is test_row_0/C:col10/1731523086183/Put/seqid=0 2024-11-13T18:38:07,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742196_1372 (size=12527) 2024-11-13T18:38:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:07,645 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:07,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:38:07,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1ba4452e799477ebeb576acefdd8ec2_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523087644/Put/seqid=0 2024-11-13T18:38:07,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523147666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742197_1373 (size=14794) 2024-11-13T18:38:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523147667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523147668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523147669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523147784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523147794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523147794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523147794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:07,899 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#A#compaction#316 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:07,899 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/436e31ddec95499b9f5d79f0f631f591 is 175, key is test_row_0/A:col10/1731523086183/Put/seqid=0 2024-11-13T18:38:07,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742198_1374 (size=31481) 2024-11-13T18:38:07,918 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/436e31ddec95499b9f5d79f0f631f591 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/436e31ddec95499b9f5d79f0f631f591 2024-11-13T18:38:07,931 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a70474ff27654302c682d5d28687cb3d/A of a70474ff27654302c682d5d28687cb3d into 436e31ddec95499b9f5d79f0f631f591(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:07,931 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:07,931 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/A, priority=12, startTime=1731523087408; duration=0sec 2024-11-13T18:38:07,931 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:07,931 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:07,955 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/da850c7774954c5ab7ca7ee8d87ec9e6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/da850c7774954c5ab7ca7ee8d87ec9e6 2024-11-13T18:38:07,960 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a70474ff27654302c682d5d28687cb3d/C of a70474ff27654302c682d5d28687cb3d into da850c7774954c5ab7ca7ee8d87ec9e6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:07,960 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:07,960 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/C, priority=12, startTime=1731523087408; duration=0sec 2024-11-13T18:38:07,961 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:07,961 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523148003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523148003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523148014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523148014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,081 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:08,090 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1ba4452e799477ebeb576acefdd8ec2_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1ba4452e799477ebeb576acefdd8ec2_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:08,095 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/e9284286a7e948229b786b77db53ab21, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:08,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/e9284286a7e948229b786b77db53ab21 is 175, key is test_row_0/A:col10/1731523087644/Put/seqid=0 2024-11-13T18:38:08,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742199_1375 (size=39749) 2024-11-13T18:38:08,112 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/e9284286a7e948229b786b77db53ab21 2024-11-13T18:38:08,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/15f72423d4144d4cb97d92462a69ebd2 is 50, key is test_row_0/B:col10/1731523087644/Put/seqid=0 2024-11-13T18:38:08,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742200_1376 (size=12151) 2024-11-13T18:38:08,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523148313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523148313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523148323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523148336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/15f72423d4144d4cb97d92462a69ebd2 2024-11-13T18:38:08,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/495511d7ab3d447fb5dabe9199ceb964 is 50, key is test_row_0/C:col10/1731523087644/Put/seqid=0 2024-11-13T18:38:08,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742201_1377 (size=12151) 2024-11-13T18:38:08,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523148823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523148823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523148844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523148844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:08,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/495511d7ab3d447fb5dabe9199ceb964 2024-11-13T18:38:09,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/e9284286a7e948229b786b77db53ab21 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e9284286a7e948229b786b77db53ab21 2024-11-13T18:38:09,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e9284286a7e948229b786b77db53ab21, entries=200, sequenceid=197, filesize=38.8 K 2024-11-13T18:38:09,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/15f72423d4144d4cb97d92462a69ebd2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/15f72423d4144d4cb97d92462a69ebd2 2024-11-13T18:38:09,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/15f72423d4144d4cb97d92462a69ebd2, entries=150, sequenceid=197, filesize=11.9 K 2024-11-13T18:38:09,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/495511d7ab3d447fb5dabe9199ceb964 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/495511d7ab3d447fb5dabe9199ceb964 2024-11-13T18:38:09,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/495511d7ab3d447fb5dabe9199ceb964, entries=150, sequenceid=197, filesize=11.9 K 2024-11-13T18:38:09,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a70474ff27654302c682d5d28687cb3d in 1387ms, sequenceid=197, compaction requested=false 2024-11-13T18:38:09,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:09,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:09,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:38:09,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d7eb5a507bdc486a9d9cfe05fec5b7fa_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523087664/Put/seqid=0 2024-11-13T18:38:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742202_1378 (size=14794) 2024-11-13T18:38:09,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:09,496 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d7eb5a507bdc486a9d9cfe05fec5b7fa_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d7eb5a507bdc486a9d9cfe05fec5b7fa_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:09,497 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/ff32448567354d68a1f7142b93e7c529, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:09,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/ff32448567354d68a1f7142b93e7c529 is 175, key is test_row_0/A:col10/1731523087664/Put/seqid=0 2024-11-13T18:38:09,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742203_1379 (size=39749) 2024-11-13T18:38:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523149694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:09,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523149813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:09,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523149843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:09,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523149844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:09,859 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_1' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:09,859 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:09,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523149863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:09,876 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:09,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523149873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:09,883 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in a70474ff27654302c682d5d28687cb3d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T18:38:09,934 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:09,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-13T18:38:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:38:09,937 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:09,937 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:09,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:09,940 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/ff32448567354d68a1f7142b93e7c529 2024-11-13T18:38:09,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/64d3fd6ea94245158e2b57e5069e438e is 50, key is test_row_0/B:col10/1731523087664/Put/seqid=0 2024-11-13T18:38:09,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742204_1380 (size=12151) 2024-11-13T18:38:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523150024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:38:10,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:38:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:38:10,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:10,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:38:10,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523150344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:10,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/64d3fd6ea94245158e2b57e5069e438e 2024-11-13T18:38:10,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/9dfc35289c8f46229692c42871390680 is 50, key is test_row_0/C:col10/1731523087664/Put/seqid=0 2024-11-13T18:38:10,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742205_1381 (size=12151) 2024-11-13T18:38:10,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:38:10,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:10,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:38:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:38:10,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:38:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:10,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/9dfc35289c8f46229692c42871390680 2024-11-13T18:38:10,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/ff32448567354d68a1f7142b93e7c529 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/ff32448567354d68a1f7142b93e7c529 2024-11-13T18:38:10,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/ff32448567354d68a1f7142b93e7c529, entries=200, sequenceid=210, filesize=38.8 K 2024-11-13T18:38:10,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/64d3fd6ea94245158e2b57e5069e438e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/64d3fd6ea94245158e2b57e5069e438e 2024-11-13T18:38:10,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/64d3fd6ea94245158e2b57e5069e438e, entries=150, sequenceid=210, filesize=11.9 K 2024-11-13T18:38:10,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/9dfc35289c8f46229692c42871390680 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9dfc35289c8f46229692c42871390680 2024-11-13T18:38:10,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9dfc35289c8f46229692c42871390680, entries=150, sequenceid=210, filesize=11.9 K 2024-11-13T18:38:10,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a70474ff27654302c682d5d28687cb3d in 1364ms, sequenceid=210, compaction requested=true 2024-11-13T18:38:10,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:10,825 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:10,826 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:10,826 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/A is initiating minor compaction (all files) 2024-11-13T18:38:10,827 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/A in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,827 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/436e31ddec95499b9f5d79f0f631f591, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e9284286a7e948229b786b77db53ab21, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/ff32448567354d68a1f7142b93e7c529] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=108.4 K 2024-11-13T18:38:10,827 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,827 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/436e31ddec95499b9f5d79f0f631f591, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e9284286a7e948229b786b77db53ab21, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/ff32448567354d68a1f7142b93e7c529] 2024-11-13T18:38:10,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:10,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:10,827 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:10,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:10,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:10,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:10,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:10,829 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 436e31ddec95499b9f5d79f0f631f591, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523086183 2024-11-13T18:38:10,830 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:10,830 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/B is initiating minor compaction (all files) 2024-11-13T18:38:10,830 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/B in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,830 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/98188398ef804fecb435417e80c5dd32, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/15f72423d4144d4cb97d92462a69ebd2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/64d3fd6ea94245158e2b57e5069e438e] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=36.0 K 2024-11-13T18:38:10,830 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e9284286a7e948229b786b77db53ab21, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731523086434 2024-11-13T18:38:10,831 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ff32448567354d68a1f7142b93e7c529, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523087664 2024-11-13T18:38:10,831 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98188398ef804fecb435417e80c5dd32, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523086183 2024-11-13T18:38:10,831 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15f72423d4144d4cb97d92462a69ebd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731523086447 2024-11-13T18:38:10,832 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d3fd6ea94245158e2b57e5069e438e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523087664 2024-11-13T18:38:10,843 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:10,844 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/5fe94ff235434573a48029aca1cf0147 is 50, key is test_row_0/B:col10/1731523087664/Put/seqid=0 2024-11-13T18:38:10,852 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:10,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T18:38:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,859 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:38:10,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:10,877 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111327a03dcb01bb49b2bc57f322c77b2fb2_a70474ff27654302c682d5d28687cb3d store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:10,879 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111327a03dcb01bb49b2bc57f322c77b2fb2_a70474ff27654302c682d5d28687cb3d, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:10,879 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111327a03dcb01bb49b2bc57f322c77b2fb2_a70474ff27654302c682d5d28687cb3d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cfc45ac918e24f308cc459218830b95f_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523089664/Put/seqid=0 2024-11-13T18:38:10,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742206_1382 (size=12629) 2024-11-13T18:38:10,906 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/5fe94ff235434573a48029aca1cf0147 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/5fe94ff235434573a48029aca1cf0147 2024-11-13T18:38:10,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742207_1383 (size=4469) 2024-11-13T18:38:10,914 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/B of a70474ff27654302c682d5d28687cb3d into 5fe94ff235434573a48029aca1cf0147(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:10,914 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:10,914 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/B, priority=13, startTime=1731523090827; duration=0sec 2024-11-13T18:38:10,914 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:10,914 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:10,914 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:10,915 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#A#compaction#325 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:10,915 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:10,915 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/C is initiating minor compaction (all files) 2024-11-13T18:38:10,915 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/C in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:10,915 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/c331954073944e66b3406d4a6095479a is 175, key is test_row_0/A:col10/1731523087664/Put/seqid=0 2024-11-13T18:38:10,915 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/da850c7774954c5ab7ca7ee8d87ec9e6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/495511d7ab3d447fb5dabe9199ceb964, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9dfc35289c8f46229692c42871390680] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=36.0 K 2024-11-13T18:38:10,915 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting da850c7774954c5ab7ca7ee8d87ec9e6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523086183 2024-11-13T18:38:10,916 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 495511d7ab3d447fb5dabe9199ceb964, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731523086447 2024-11-13T18:38:10,916 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dfc35289c8f46229692c42871390680, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523087664 2024-11-13T18:38:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742208_1384 (size=12304) 2024-11-13T18:38:10,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:10,926 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:10,926 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/9a350eb0a82a4294a6cc987ba1c28374 is 50, key is test_row_0/C:col10/1731523087664/Put/seqid=0 2024-11-13T18:38:10,927 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cfc45ac918e24f308cc459218830b95f_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cfc45ac918e24f308cc459218830b95f_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/df7213ad4d2b4e1289af82c6cf1808da, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:10,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/df7213ad4d2b4e1289af82c6cf1808da is 175, key is test_row_0/A:col10/1731523089664/Put/seqid=0 2024-11-13T18:38:10,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742209_1385 (size=31583) 2024-11-13T18:38:10,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742210_1386 (size=12629) 2024-11-13T18:38:10,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742211_1387 (size=31105) 2024-11-13T18:38:10,942 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/c331954073944e66b3406d4a6095479a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/c331954073944e66b3406d4a6095479a 2024-11-13T18:38:10,943 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/9a350eb0a82a4294a6cc987ba1c28374 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9a350eb0a82a4294a6cc987ba1c28374 2024-11-13T18:38:10,947 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/A of a70474ff27654302c682d5d28687cb3d into c331954073944e66b3406d4a6095479a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:10,947 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:10,947 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/A, priority=13, startTime=1731523090825; duration=0sec 2024-11-13T18:38:10,947 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:10,947 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:10,948 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/C of a70474ff27654302c682d5d28687cb3d into 9a350eb0a82a4294a6cc987ba1c28374(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:10,948 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:10,948 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/C, priority=13, startTime=1731523090827; duration=0sec 2024-11-13T18:38:10,948 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:10,948 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523150964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:38:11,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523151073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:11,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523151293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:11,338 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/df7213ad4d2b4e1289af82c6cf1808da 2024-11-13T18:38:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/4f37c1875ee940be8dfb68d8939f6f49 is 50, key is test_row_0/B:col10/1731523089664/Put/seqid=0 2024-11-13T18:38:11,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742212_1388 (size=12151) 2024-11-13T18:38:11,393 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/4f37c1875ee940be8dfb68d8939f6f49 2024-11-13T18:38:11,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a is 50, key is test_row_0/C:col10/1731523089664/Put/seqid=0 2024-11-13T18:38:11,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742213_1389 (size=12151) 2024-11-13T18:38:11,456 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a 2024-11-13T18:38:11,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/df7213ad4d2b4e1289af82c6cf1808da as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/df7213ad4d2b4e1289af82c6cf1808da 2024-11-13T18:38:11,464 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/df7213ad4d2b4e1289af82c6cf1808da, entries=150, sequenceid=235, filesize=30.4 K 2024-11-13T18:38:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/4f37c1875ee940be8dfb68d8939f6f49 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/4f37c1875ee940be8dfb68d8939f6f49 2024-11-13T18:38:11,468 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/4f37c1875ee940be8dfb68d8939f6f49, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T18:38:11,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a 2024-11-13T18:38:11,476 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T18:38:11,477 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a70474ff27654302c682d5d28687cb3d in 618ms, sequenceid=235, compaction requested=false 2024-11-13T18:38:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-13T18:38:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-13T18:38:11,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T18:38:11,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5400 sec 2024-11-13T18:38:11,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.5480 sec 2024-11-13T18:38:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:11,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:38:11,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a2d91566028948a1a15fafe9ce9e95da_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523091617/Put/seqid=0 2024-11-13T18:38:11,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742214_1390 (size=14794) 2024-11-13T18:38:11,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a2d91566028948a1a15fafe9ce9e95da_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a2d91566028948a1a15fafe9ce9e95da_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:11,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/555f449102814b0698c404e1228f734b, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:11,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/555f449102814b0698c404e1228f734b is 175, key is test_row_0/A:col10/1731523091617/Put/seqid=0 2024-11-13T18:38:11,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742215_1391 (size=39749) 2024-11-13T18:38:11,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/555f449102814b0698c404e1228f734b 2024-11-13T18:38:11,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/8d242fd2109c440691268630f1c3b540 is 50, key is test_row_0/B:col10/1731523091617/Put/seqid=0 2024-11-13T18:38:11,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742216_1392 (size=12151) 2024-11-13T18:38:11,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/8d242fd2109c440691268630f1c3b540 2024-11-13T18:38:11,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e5dca9e898b3432ba81188107d899057 is 50, key is test_row_0/C:col10/1731523091617/Put/seqid=0 2024-11-13T18:38:11,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742217_1393 (size=12151) 2024-11-13T18:38:11,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e5dca9e898b3432ba81188107d899057 2024-11-13T18:38:11,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/555f449102814b0698c404e1228f734b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/555f449102814b0698c404e1228f734b 2024-11-13T18:38:11,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/555f449102814b0698c404e1228f734b, entries=200, sequenceid=250, filesize=38.8 K 2024-11-13T18:38:11,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/8d242fd2109c440691268630f1c3b540 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/8d242fd2109c440691268630f1c3b540 2024-11-13T18:38:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/8d242fd2109c440691268630f1c3b540, entries=150, sequenceid=250, filesize=11.9 K 2024-11-13T18:38:11,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/e5dca9e898b3432ba81188107d899057 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e5dca9e898b3432ba81188107d899057 2024-11-13T18:38:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e5dca9e898b3432ba81188107d899057, entries=150, sequenceid=250, filesize=11.9 K 2024-11-13T18:38:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=67.09 KB/68700 for a70474ff27654302c682d5d28687cb3d in 117ms, sequenceid=250, compaction requested=true 2024-11-13T18:38:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:11,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:11,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:11,735 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,735 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,736 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:11,736 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:11,737 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/B is initiating minor compaction (all files) 2024-11-13T18:38:11,737 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/A is initiating minor compaction (all files) 2024-11-13T18:38:11,737 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/B in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:11,737 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/A in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,737 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/c331954073944e66b3406d4a6095479a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/df7213ad4d2b4e1289af82c6cf1808da, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/555f449102814b0698c404e1228f734b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=100.0 K 2024-11-13T18:38:11,737 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/5fe94ff235434573a48029aca1cf0147, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/4f37c1875ee940be8dfb68d8939f6f49, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/8d242fd2109c440691268630f1c3b540] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=36.1 K 2024-11-13T18:38:11,737 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:11,737 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/c331954073944e66b3406d4a6095479a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/df7213ad4d2b4e1289af82c6cf1808da, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/555f449102814b0698c404e1228f734b] 2024-11-13T18:38:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,737 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c331954073944e66b3406d4a6095479a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523087664 2024-11-13T18:38:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,737 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fe94ff235434573a48029aca1cf0147, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523087664 2024-11-13T18:38:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,738 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting df7213ad4d2b4e1289af82c6cf1808da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731523089539 2024-11-13T18:38:11,738 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f37c1875ee940be8dfb68d8939f6f49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731523089539 2024-11-13T18:38:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,738 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d242fd2109c440691268630f1c3b540, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731523090933 2024-11-13T18:38:11,738 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 555f449102814b0698c404e1228f734b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731523090933 2024-11-13T18:38:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,744 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,746 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,747 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/b5e53dcb4d274024a58078f55f52871b is 50, key is test_row_0/B:col10/1731523091617/Put/seqid=0 2024-11-13T18:38:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,750 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113044fbc61d3584698ad4482bb8bce2128_a70474ff27654302c682d5d28687cb3d store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,751 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113044fbc61d3584698ad4482bb8bce2128_a70474ff27654302c682d5d28687cb3d, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,751 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113044fbc61d3584698ad4482bb8bce2128_a70474ff27654302c682d5d28687cb3d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742218_1394 (size=12731) 2024-11-13T18:38:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742219_1395 (size=4469) 2024-11-13T18:38:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:11,794 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:11,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T18:38:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113865a428f721d4be7bb6a4e5674105691_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523091657/Put/seqid=0 2024-11-13T18:38:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742220_1396 (size=12454) 2024-11-13T18:38:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,808 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,812 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113865a428f721d4be7bb6a4e5674105691_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113865a428f721d4be7bb6a4e5674105691_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,813 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8b25c5ef367e410f98ec940d2afb3440, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8b25c5ef367e410f98ec940d2afb3440 is 175, key is test_row_0/A:col10/1731523091657/Put/seqid=0 2024-11-13T18:38:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742221_1397 (size=31251) 2024-11-13T18:38:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8b25c5ef367e410f98ec940d2afb3440 2024-11-13T18:38:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/1426902700d54fbb8e1374ec10fd42ee is 50, key is test_row_0/B:col10/1731523091657/Put/seqid=0 2024-11-13T18:38:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742222_1398 (size=12297) 2024-11-13T18:38:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/1426902700d54fbb8e1374ec10fd42ee 2024-11-13T18:38:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/5b789fb2942c43dca05aba794dc3cda2 is 50, key is test_row_0/C:col10/1731523091657/Put/seqid=0 2024-11-13T18:38:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742223_1399 (size=9857) 2024-11-13T18:38:11,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/5b789fb2942c43dca05aba794dc3cda2 2024-11-13T18:38:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/8b25c5ef367e410f98ec940d2afb3440 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8b25c5ef367e410f98ec940d2afb3440 2024-11-13T18:38:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8b25c5ef367e410f98ec940d2afb3440, entries=150, sequenceid=264, filesize=30.5 K 2024-11-13T18:38:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/1426902700d54fbb8e1374ec10fd42ee as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1426902700d54fbb8e1374ec10fd42ee 2024-11-13T18:38:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1426902700d54fbb8e1374ec10fd42ee, entries=150, sequenceid=264, filesize=12.0 K 2024-11-13T18:38:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/5b789fb2942c43dca05aba794dc3cda2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/5b789fb2942c43dca05aba794dc3cda2 2024-11-13T18:38:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/5b789fb2942c43dca05aba794dc3cda2, entries=100, sequenceid=264, filesize=9.6 K 2024-11-13T18:38:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=87.22 KB/89310 for a70474ff27654302c682d5d28687cb3d in 162ms, sequenceid=264, compaction requested=true 2024-11-13T18:38:11,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:11,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:11,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:38:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:11,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T18:38:11,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:11,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T18:38:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:11,968 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:11,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T18:38:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:12,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dc1f51330be144a68ff2a73bfa345c8a_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523091914/Put/seqid=0 2024-11-13T18:38:12,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742225_1401 (size=42942) 2024-11-13T18:38:12,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523152021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523152029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523152030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523152030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523152037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T18:38:12,084 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:12,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-13T18:38:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:38:12,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:12,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:12,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:12,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523152143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523152143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,161 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/b5e53dcb4d274024a58078f55f52871b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b5e53dcb4d274024a58078f55f52871b 2024-11-13T18:38:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523152154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523152154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,165 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/B of a70474ff27654302c682d5d28687cb3d into b5e53dcb4d274024a58078f55f52871b(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:12,165 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:12,165 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/B, priority=13, startTime=1731523091735; duration=0sec 2024-11-13T18:38:12,165 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T18:38:12,165 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:12,165 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-13T18:38:12,165 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:38:12,166 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:38:12,166 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. because compaction request was cancelled 2024-11-13T18:38:12,166 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:12,166 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:12,166 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46788 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:12,167 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/C is initiating minor compaction (all files) 2024-11-13T18:38:12,167 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/C in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,167 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9a350eb0a82a4294a6cc987ba1c28374, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e5dca9e898b3432ba81188107d899057, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/5b789fb2942c43dca05aba794dc3cda2] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=45.7 K 2024-11-13T18:38:12,167 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a350eb0a82a4294a6cc987ba1c28374, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523087664 2024-11-13T18:38:12,167 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c2c6326bdfc4e91a16ea6f0b5cedd7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731523089539 2024-11-13T18:38:12,167 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e5dca9e898b3432ba81188107d899057, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731523090933 2024-11-13T18:38:12,168 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b789fb2942c43dca05aba794dc3cda2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731523091657 2024-11-13T18:38:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523152163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,175 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:12,175 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/7ca6e332e4184714bc8a4a492b4c579f is 50, key is test_row_0/C:col10/1731523091657/Put/seqid=0 2024-11-13T18:38:12,177 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#A#compaction#333 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:12,177 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/03e3325779bc44a7a7ebe2bea1f9effe is 175, key is test_row_0/A:col10/1731523091617/Put/seqid=0 2024-11-13T18:38:12,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742226_1402 (size=12865) 2024-11-13T18:38:12,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742227_1403 (size=31685) 2024-11-13T18:38:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:38:12,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:38:12,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:12,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523152363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523152364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523152373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523152373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:38:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523152383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:38:12,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:12,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dc1f51330be144a68ff2a73bfa345c8a_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc1f51330be144a68ff2a73bfa345c8a_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:12,422 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/a2f2e9d5f0714eb78afc86a7ce25f905, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:12,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/a2f2e9d5f0714eb78afc86a7ce25f905 is 175, key is test_row_0/A:col10/1731523091914/Put/seqid=0 2024-11-13T18:38:12,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742224_1400 (size=135636) 2024-11-13T18:38:12,425 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/a2f2e9d5f0714eb78afc86a7ce25f905 2024-11-13T18:38:12,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/b325f881e2a749eeb2351966fc0eeae0 is 50, key is test_row_0/B:col10/1731523091914/Put/seqid=0 2024-11-13T18:38:12,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742228_1404 (size=12301) 2024-11-13T18:38:12,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:38:12,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:12,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,585 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/7ca6e332e4184714bc8a4a492b4c579f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/7ca6e332e4184714bc8a4a492b4c579f 2024-11-13T18:38:12,587 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/03e3325779bc44a7a7ebe2bea1f9effe as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/03e3325779bc44a7a7ebe2bea1f9effe 2024-11-13T18:38:12,589 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a70474ff27654302c682d5d28687cb3d/C of a70474ff27654302c682d5d28687cb3d into 7ca6e332e4184714bc8a4a492b4c579f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:12,589 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:12,589 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/C, priority=12, startTime=1731523091956; duration=0sec 2024-11-13T18:38:12,589 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T18:38:12,589 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:12,589 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:12,589 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:12,589 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:38:12,590 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:38:12,590 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:38:12,590 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. because compaction request was cancelled 2024-11-13T18:38:12,590 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:12,590 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T18:38:12,590 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:38:12,591 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:38:12,591 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. because compaction request was cancelled 2024-11-13T18:38:12,591 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:12,592 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/A of a70474ff27654302c682d5d28687cb3d into 03e3325779bc44a7a7ebe2bea1f9effe(size=30.9 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:12,592 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:12,592 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/A, priority=13, startTime=1731523091735; duration=0sec 2024-11-13T18:38:12,592 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:12,592 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:12,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523152673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523152673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:38:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523152693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523152693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:38:12,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523152703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:12,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/b325f881e2a749eeb2351966fc0eeae0 2024-11-13T18:38:12,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/c8cd20ab18d34a96aeb3a909a483b568 is 50, key is test_row_0/C:col10/1731523091914/Put/seqid=0 2024-11-13T18:38:12,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:38:12,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:12,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742229_1405 (size=12301) 2024-11-13T18:38:12,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/c8cd20ab18d34a96aeb3a909a483b568 2024-11-13T18:38:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:12,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/a2f2e9d5f0714eb78afc86a7ce25f905 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a2f2e9d5f0714eb78afc86a7ce25f905 2024-11-13T18:38:12,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a2f2e9d5f0714eb78afc86a7ce25f905, entries=750, sequenceid=282, filesize=132.5 K 2024-11-13T18:38:12,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/b325f881e2a749eeb2351966fc0eeae0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b325f881e2a749eeb2351966fc0eeae0 2024-11-13T18:38:12,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b325f881e2a749eeb2351966fc0eeae0, entries=150, sequenceid=282, filesize=12.0 K 2024-11-13T18:38:12,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/c8cd20ab18d34a96aeb3a909a483b568 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/c8cd20ab18d34a96aeb3a909a483b568 2024-11-13T18:38:12,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/c8cd20ab18d34a96aeb3a909a483b568, entries=150, sequenceid=282, filesize=12.0 K 2024-11-13T18:38:12,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for a70474ff27654302c682d5d28687cb3d in 902ms, sequenceid=282, compaction requested=true 2024-11-13T18:38:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:12,871 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:12,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:12,871 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:12,872 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 198572 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:12,872 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/A is initiating minor compaction (all files) 2024-11-13T18:38:12,872 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/A in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,872 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/03e3325779bc44a7a7ebe2bea1f9effe, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8b25c5ef367e410f98ec940d2afb3440, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a2f2e9d5f0714eb78afc86a7ce25f905] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=193.9 K 2024-11-13T18:38:12,872 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,872 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/03e3325779bc44a7a7ebe2bea1f9effe, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8b25c5ef367e410f98ec940d2afb3440, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a2f2e9d5f0714eb78afc86a7ce25f905] 2024-11-13T18:38:12,872 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:12,872 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/B is initiating minor compaction (all files) 2024-11-13T18:38:12,872 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/B in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:12,872 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b5e53dcb4d274024a58078f55f52871b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1426902700d54fbb8e1374ec10fd42ee, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b325f881e2a749eeb2351966fc0eeae0] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=36.5 K 2024-11-13T18:38:12,873 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e3325779bc44a7a7ebe2bea1f9effe, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731523090933 2024-11-13T18:38:12,873 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e53dcb4d274024a58078f55f52871b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731523090933 2024-11-13T18:38:12,873 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b25c5ef367e410f98ec940d2afb3440, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731523091657 2024-11-13T18:38:12,873 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1426902700d54fbb8e1374ec10fd42ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731523091657 2024-11-13T18:38:12,873 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f2e9d5f0714eb78afc86a7ce25f905, keycount=750, bloomtype=ROW, size=132.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731523091879 2024-11-13T18:38:12,874 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b325f881e2a749eeb2351966fc0eeae0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731523091914 2024-11-13T18:38:12,882 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:12,882 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:12,883 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/77533df3946149a59537128b50ea9181 is 50, key is test_row_0/B:col10/1731523091914/Put/seqid=0 2024-11-13T18:38:12,885 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cc5ca898fce54c0f8fece469a1afb37b_a70474ff27654302c682d5d28687cb3d store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:12,891 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cc5ca898fce54c0f8fece469a1afb37b_a70474ff27654302c682d5d28687cb3d, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:12,891 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cc5ca898fce54c0f8fece469a1afb37b_a70474ff27654302c682d5d28687cb3d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:12,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742230_1406 (size=12983) 2024-11-13T18:38:12,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742231_1407 (size=4469) 2024-11-13T18:38:12,911 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#A#compaction#343 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:12,912 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/6f6be1446b754d5987c1b99b42a9a81f is 175, key is test_row_0/A:col10/1731523091914/Put/seqid=0 2024-11-13T18:38:12,918 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/77533df3946149a59537128b50ea9181 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/77533df3946149a59537128b50ea9181 2024-11-13T18:38:12,923 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/B of a70474ff27654302c682d5d28687cb3d into 77533df3946149a59537128b50ea9181(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:12,923 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:12,923 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/B, priority=13, startTime=1731523092871; duration=0sec 2024-11-13T18:38:12,923 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:12,923 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:12,923 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:38:12,924 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T18:38:12,924 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T18:38:12,924 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. because compaction request was cancelled 2024-11-13T18:38:12,924 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:12,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742232_1408 (size=31937) 2024-11-13T18:38:12,930 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/6f6be1446b754d5987c1b99b42a9a81f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6f6be1446b754d5987c1b99b42a9a81f 2024-11-13T18:38:12,934 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/A of a70474ff27654302c682d5d28687cb3d into 6f6be1446b754d5987c1b99b42a9a81f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:12,934 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:12,934 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/A, priority=13, startTime=1731523092871; duration=0sec 2024-11-13T18:38:12,934 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:12,934 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:13,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T18:38:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:13,005 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T18:38:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111318baf14c549a45d4af2a9950d4ee1278_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523092027/Put/seqid=0 2024-11-13T18:38:13,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742233_1409 (size=12454) 2024-11-13T18:38:13,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:38:13,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523153242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523153243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523153244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523153245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523153245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523153354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523153363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523153363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523153363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523153363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:13,422 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111318baf14c549a45d4af2a9950d4ee1278_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111318baf14c549a45d4af2a9950d4ee1278_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/1058b22e5e3549caacfbb15f32ff51bb, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/1058b22e5e3549caacfbb15f32ff51bb is 175, key is test_row_0/A:col10/1731523092027/Put/seqid=0 2024-11-13T18:38:13,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742234_1410 (size=31255) 2024-11-13T18:38:13,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523153573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523153583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523153583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523153583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523153583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,827 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/1058b22e5e3549caacfbb15f32ff51bb 2024-11-13T18:38:13,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/c525c2346e50410b90b6246059d1d085 is 50, key is test_row_0/B:col10/1731523092027/Put/seqid=0 2024-11-13T18:38:13,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742235_1411 (size=12301) 2024-11-13T18:38:13,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:38:13,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523153893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523153894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523153894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523153894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523153894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:38:14,239 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/c525c2346e50410b90b6246059d1d085 2024-11-13T18:38:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/2664cd6de6374513a7faf2e1d68d13f9 is 50, key is test_row_0/C:col10/1731523092027/Put/seqid=0 2024-11-13T18:38:14,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742236_1412 (size=12301) 2024-11-13T18:38:14,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523154414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523154414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523154414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:14,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523154414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:14,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523154414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:14,651 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/2664cd6de6374513a7faf2e1d68d13f9 2024-11-13T18:38:14,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/1058b22e5e3549caacfbb15f32ff51bb as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1058b22e5e3549caacfbb15f32ff51bb 2024-11-13T18:38:14,659 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1058b22e5e3549caacfbb15f32ff51bb, entries=150, sequenceid=305, filesize=30.5 K 2024-11-13T18:38:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/c525c2346e50410b90b6246059d1d085 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/c525c2346e50410b90b6246059d1d085 2024-11-13T18:38:14,664 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/c525c2346e50410b90b6246059d1d085, entries=150, sequenceid=305, filesize=12.0 K 2024-11-13T18:38:14,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/2664cd6de6374513a7faf2e1d68d13f9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2664cd6de6374513a7faf2e1d68d13f9 2024-11-13T18:38:14,668 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2664cd6de6374513a7faf2e1d68d13f9, entries=150, sequenceid=305, filesize=12.0 K 2024-11-13T18:38:14,669 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for a70474ff27654302c682d5d28687cb3d in 1664ms, sequenceid=305, compaction requested=true 2024-11-13T18:38:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-13T18:38:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-13T18:38:14,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T18:38:14,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5840 sec 2024-11-13T18:38:14,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.5880 sec 2024-11-13T18:38:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:15,439 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:15,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T18:38:15,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137b91738b9a544dc6875f89590fd35f9d_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523093244/Put/seqid=0 2024-11-13T18:38:15,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742237_1413 (size=14994) 2024-11-13T18:38:15,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523155456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523155457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523155466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523155467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523155468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523155574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523155574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523155574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523155583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523155583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523155783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523155783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523155783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523155793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523155793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:15,851 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:15,857 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137b91738b9a544dc6875f89590fd35f9d_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b91738b9a544dc6875f89590fd35f9d_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:15,858 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/a1acd50074f64503acab71d5bf5b78f2, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:15,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/a1acd50074f64503acab71d5bf5b78f2 is 175, key is test_row_0/A:col10/1731523093244/Put/seqid=0 2024-11-13T18:38:15,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742238_1414 (size=39949) 2024-11-13T18:38:15,863 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/a1acd50074f64503acab71d5bf5b78f2 2024-11-13T18:38:15,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/69c7e0f2600d4da2bab48ef313437b25 is 50, key is test_row_0/B:col10/1731523093244/Put/seqid=0 2024-11-13T18:38:15,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742239_1415 (size=12301) 2024-11-13T18:38:15,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/69c7e0f2600d4da2bab48ef313437b25 2024-11-13T18:38:15,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/69eb488e6b804611b673b5301f29ffad is 50, key is test_row_0/C:col10/1731523093244/Put/seqid=0 2024-11-13T18:38:15,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742240_1416 (size=12301) 2024-11-13T18:38:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523156103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523156103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523156103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523156103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523156103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T18:38:16,254 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:16,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-13T18:38:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:38:16,257 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:16,258 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:16,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:16,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/69eb488e6b804611b673b5301f29ffad 2024-11-13T18:38:16,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/a1acd50074f64503acab71d5bf5b78f2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a1acd50074f64503acab71d5bf5b78f2 2024-11-13T18:38:16,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a1acd50074f64503acab71d5bf5b78f2, entries=200, sequenceid=325, filesize=39.0 K 2024-11-13T18:38:16,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/69c7e0f2600d4da2bab48ef313437b25 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69c7e0f2600d4da2bab48ef313437b25 2024-11-13T18:38:16,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69c7e0f2600d4da2bab48ef313437b25, entries=150, sequenceid=325, filesize=12.0 K 2024-11-13T18:38:16,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/69eb488e6b804611b673b5301f29ffad as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/69eb488e6b804611b673b5301f29ffad 2024-11-13T18:38:16,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/69eb488e6b804611b673b5301f29ffad, entries=150, sequenceid=325, filesize=12.0 K 2024-11-13T18:38:16,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a70474ff27654302c682d5d28687cb3d in 864ms, sequenceid=325, compaction requested=true 2024-11-13T18:38:16,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:16,303 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:16,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:16,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:16,304 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:16,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:16,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:16,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:16,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:16,306 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:16,306 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/A is initiating minor compaction (all files) 2024-11-13T18:38:16,306 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/A in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:16,306 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6f6be1446b754d5987c1b99b42a9a81f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1058b22e5e3549caacfbb15f32ff51bb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a1acd50074f64503acab71d5bf5b78f2] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=100.7 K 2024-11-13T18:38:16,306 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:16,306 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6f6be1446b754d5987c1b99b42a9a81f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1058b22e5e3549caacfbb15f32ff51bb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a1acd50074f64503acab71d5bf5b78f2] 2024-11-13T18:38:16,308 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f6be1446b754d5987c1b99b42a9a81f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731523091914 2024-11-13T18:38:16,308 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:16,308 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/B is initiating minor compaction (all files) 2024-11-13T18:38:16,309 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/B in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:16,309 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/77533df3946149a59537128b50ea9181, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/c525c2346e50410b90b6246059d1d085, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69c7e0f2600d4da2bab48ef313437b25] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=36.7 K 2024-11-13T18:38:16,309 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1058b22e5e3549caacfbb15f32ff51bb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731523092027 2024-11-13T18:38:16,310 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 77533df3946149a59537128b50ea9181, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731523091914 2024-11-13T18:38:16,310 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1acd50074f64503acab71d5bf5b78f2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731523093240 2024-11-13T18:38:16,311 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c525c2346e50410b90b6246059d1d085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731523092027 2024-11-13T18:38:16,312 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c7e0f2600d4da2bab48ef313437b25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731523093243 2024-11-13T18:38:16,320 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:16,326 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113b97059c498fa4c3cb025548e18bb8321_a70474ff27654302c682d5d28687cb3d store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:16,327 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:16,328 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/f8afa29f2e9142519158e90d620a4993 is 50, key is test_row_0/B:col10/1731523093244/Put/seqid=0 2024-11-13T18:38:16,328 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113b97059c498fa4c3cb025548e18bb8321_a70474ff27654302c682d5d28687cb3d, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:16,328 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b97059c498fa4c3cb025548e18bb8321_a70474ff27654302c682d5d28687cb3d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:16,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742241_1417 (size=13085) 2024-11-13T18:38:16,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742242_1418 (size=4469) 2024-11-13T18:38:16,342 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#A#compaction#350 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:16,343 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/597564033dda4c919c00383ffae290f7 is 175, key is test_row_0/A:col10/1731523093244/Put/seqid=0 2024-11-13T18:38:16,343 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/f8afa29f2e9142519158e90d620a4993 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/f8afa29f2e9142519158e90d620a4993 2024-11-13T18:38:16,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742243_1419 (size=32039) 2024-11-13T18:38:16,349 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/B of a70474ff27654302c682d5d28687cb3d into f8afa29f2e9142519158e90d620a4993(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:16,349 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:16,349 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/B, priority=13, startTime=1731523096304; duration=0sec 2024-11-13T18:38:16,350 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:16,350 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:16,350 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:16,351 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:16,352 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/C is initiating minor compaction (all files) 2024-11-13T18:38:16,352 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/C in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:16,352 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/7ca6e332e4184714bc8a4a492b4c579f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/c8cd20ab18d34a96aeb3a909a483b568, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2664cd6de6374513a7faf2e1d68d13f9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/69eb488e6b804611b673b5301f29ffad] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=48.6 K 2024-11-13T18:38:16,353 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca6e332e4184714bc8a4a492b4c579f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731523090933 2024-11-13T18:38:16,353 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/597564033dda4c919c00383ffae290f7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/597564033dda4c919c00383ffae290f7 2024-11-13T18:38:16,353 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c8cd20ab18d34a96aeb3a909a483b568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731523091914 2024-11-13T18:38:16,353 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2664cd6de6374513a7faf2e1d68d13f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731523092027 2024-11-13T18:38:16,354 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 69eb488e6b804611b673b5301f29ffad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731523093243 2024-11-13T18:38:16,358 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/A of a70474ff27654302c682d5d28687cb3d into 597564033dda4c919c00383ffae290f7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:16,358 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:16,358 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/A, priority=13, startTime=1731523096303; duration=0sec 2024-11-13T18:38:16,359 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:16,359 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:38:16,366 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#C#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:16,367 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/2cc78b381fef4fbb9ded05469f57dfb9 is 50, key is test_row_0/C:col10/1731523093244/Put/seqid=0 2024-11-13T18:38:16,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742244_1420 (size=13051) 2024-11-13T18:38:16,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T18:38:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:16,412 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T18:38:16,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ad53757f15ff4709b2daa875627a98d1_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523095467/Put/seqid=0 2024-11-13T18:38:16,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742245_1421 (size=12454) 2024-11-13T18:38:16,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:16,433 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ad53757f15ff4709b2daa875627a98d1_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ad53757f15ff4709b2daa875627a98d1_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/3c3afa24a55f4262a822860d4c0a8298, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:16,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/3c3afa24a55f4262a822860d4c0a8298 is 175, key is test_row_0/A:col10/1731523095467/Put/seqid=0 2024-11-13T18:38:16,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742246_1422 (size=31255) 2024-11-13T18:38:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:38:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:16,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523156660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523156667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523156669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523156670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523156672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,779 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/2cc78b381fef4fbb9ded05469f57dfb9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2cc78b381fef4fbb9ded05469f57dfb9 2024-11-13T18:38:16,783 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a70474ff27654302c682d5d28687cb3d/C of a70474ff27654302c682d5d28687cb3d into 2cc78b381fef4fbb9ded05469f57dfb9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:16,783 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:16,783 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/C, priority=12, startTime=1731523096304; duration=0sec 2024-11-13T18:38:16,783 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:16,783 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:16,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523156784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523156784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523156793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523156793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523156793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:16,846 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/3c3afa24a55f4262a822860d4c0a8298 2024-11-13T18:38:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/3ca32601fe584439bf3873e87d31b11d is 50, key is test_row_0/B:col10/1731523095467/Put/seqid=0 2024-11-13T18:38:16,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742247_1423 (size=12301) 2024-11-13T18:38:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:38:17,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523157003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523157003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523157014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523157014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523157014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,257 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/3ca32601fe584439bf3873e87d31b11d 2024-11-13T18:38:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/6c212c6d9bcc43a28ba25f78a23442e9 is 50, key is test_row_0/C:col10/1731523095467/Put/seqid=0 2024-11-13T18:38:17,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742248_1424 (size=12301) 2024-11-13T18:38:17,273 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/6c212c6d9bcc43a28ba25f78a23442e9 2024-11-13T18:38:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/3c3afa24a55f4262a822860d4c0a8298 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/3c3afa24a55f4262a822860d4c0a8298 2024-11-13T18:38:17,280 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/3c3afa24a55f4262a822860d4c0a8298, entries=150, sequenceid=343, filesize=30.5 K 2024-11-13T18:38:17,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/3ca32601fe584439bf3873e87d31b11d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ca32601fe584439bf3873e87d31b11d 2024-11-13T18:38:17,286 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ca32601fe584439bf3873e87d31b11d, entries=150, sequenceid=343, filesize=12.0 K 2024-11-13T18:38:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/6c212c6d9bcc43a28ba25f78a23442e9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c212c6d9bcc43a28ba25f78a23442e9 2024-11-13T18:38:17,290 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c212c6d9bcc43a28ba25f78a23442e9, entries=150, sequenceid=343, filesize=12.0 K 2024-11-13T18:38:17,291 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a70474ff27654302c682d5d28687cb3d in 879ms, sequenceid=343, compaction requested=false 2024-11-13T18:38:17,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:17,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-13T18:38:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-13T18:38:17,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-13T18:38:17,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0340 sec 2024-11-13T18:38:17,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0410 sec 2024-11-13T18:38:17,301 WARN [regionserver/39e84130bbc9:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-13T18:38:17,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/5bce60809a7c4685b3340349ed3fb26b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e13c21668e20451190ea0431e1bc739b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/bea4122a257545d3bc238404d89c35fa, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6be19aa8d16a47c0ab4ea70f4afe71be, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/decd69a2bebd4a74b50a92a1df6543fa, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8f00a24dd2f74b52aea1e062a7312f83, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2bc03960a6824fda8d5d1bc1189d1795, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8a1c9d55e2ff4ccb9d2464ac22bc1032, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/d079af0742f1424a8702ec0b50d5a58a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/14763a04eec2421ab9c8946ce5c07846, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/fab4f71d52204f8f991f48ac2f55cce8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/436e31ddec95499b9f5d79f0f631f591, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e9284286a7e948229b786b77db53ab21, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/ff32448567354d68a1f7142b93e7c529, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/c331954073944e66b3406d4a6095479a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/df7213ad4d2b4e1289af82c6cf1808da, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/555f449102814b0698c404e1228f734b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/03e3325779bc44a7a7ebe2bea1f9effe, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8b25c5ef367e410f98ec940d2afb3440, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a2f2e9d5f0714eb78afc86a7ce25f905, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6f6be1446b754d5987c1b99b42a9a81f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1058b22e5e3549caacfbb15f32ff51bb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a1acd50074f64503acab71d5bf5b78f2] to archive 2024-11-13T18:38:17,309 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:17,315 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/5bce60809a7c4685b3340349ed3fb26b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/5bce60809a7c4685b3340349ed3fb26b 2024-11-13T18:38:17,316 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e13c21668e20451190ea0431e1bc739b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e13c21668e20451190ea0431e1bc739b 2024-11-13T18:38:17,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/bea4122a257545d3bc238404d89c35fa to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/bea4122a257545d3bc238404d89c35fa 2024-11-13T18:38:17,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6be19aa8d16a47c0ab4ea70f4afe71be to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6be19aa8d16a47c0ab4ea70f4afe71be 2024-11-13T18:38:17,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/decd69a2bebd4a74b50a92a1df6543fa to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/decd69a2bebd4a74b50a92a1df6543fa 2024-11-13T18:38:17,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8f00a24dd2f74b52aea1e062a7312f83 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8f00a24dd2f74b52aea1e062a7312f83 2024-11-13T18:38:17,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2bc03960a6824fda8d5d1bc1189d1795 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2bc03960a6824fda8d5d1bc1189d1795 2024-11-13T18:38:17,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8a1c9d55e2ff4ccb9d2464ac22bc1032 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8a1c9d55e2ff4ccb9d2464ac22bc1032 2024-11-13T18:38:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:17,329 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:17,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T18:38:17,334 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/d079af0742f1424a8702ec0b50d5a58a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/d079af0742f1424a8702ec0b50d5a58a 2024-11-13T18:38:17,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/14763a04eec2421ab9c8946ce5c07846 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/14763a04eec2421ab9c8946ce5c07846 2024-11-13T18:38:17,355 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/fab4f71d52204f8f991f48ac2f55cce8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/fab4f71d52204f8f991f48ac2f55cce8 2024-11-13T18:38:17,356 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/436e31ddec95499b9f5d79f0f631f591 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/436e31ddec95499b9f5d79f0f631f591 2024-11-13T18:38:17,357 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e9284286a7e948229b786b77db53ab21 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/e9284286a7e948229b786b77db53ab21 2024-11-13T18:38:17,358 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/ff32448567354d68a1f7142b93e7c529 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/ff32448567354d68a1f7142b93e7c529 2024-11-13T18:38:17,361 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/c331954073944e66b3406d4a6095479a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/c331954073944e66b3406d4a6095479a 2024-11-13T18:38:17,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111314ee6378d1324cf6bb12f42157da5161_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523096670/Put/seqid=0 2024-11-13T18:38:17,364 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/df7213ad4d2b4e1289af82c6cf1808da to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/df7213ad4d2b4e1289af82c6cf1808da 2024-11-13T18:38:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,366 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/555f449102814b0698c404e1228f734b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/555f449102814b0698c404e1228f734b 2024-11-13T18:38:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523157354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,367 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/03e3325779bc44a7a7ebe2bea1f9effe to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/03e3325779bc44a7a7ebe2bea1f9effe 2024-11-13T18:38:17,369 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8b25c5ef367e410f98ec940d2afb3440 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/8b25c5ef367e410f98ec940d2afb3440 2024-11-13T18:38:17,370 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a2f2e9d5f0714eb78afc86a7ce25f905 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a2f2e9d5f0714eb78afc86a7ce25f905 2024-11-13T18:38:17,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742249_1425 (size=14994) 2024-11-13T18:38:17,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:17,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523157364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,372 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6f6be1446b754d5987c1b99b42a9a81f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/6f6be1446b754d5987c1b99b42a9a81f 2024-11-13T18:38:17,373 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1058b22e5e3549caacfbb15f32ff51bb to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1058b22e5e3549caacfbb15f32ff51bb 2024-11-13T18:38:17,374 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a1acd50074f64503acab71d5bf5b78f2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/a1acd50074f64503acab71d5bf5b78f2 2024-11-13T18:38:17,375 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111314ee6378d1324cf6bb12f42157da5161_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111314ee6378d1324cf6bb12f42157da5161_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:17,376 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/2795375abeba4a4ab8ed73e88f85438f, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:17,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/2795375abeba4a4ab8ed73e88f85438f is 175, key is test_row_0/A:col10/1731523096670/Put/seqid=0 2024-11-13T18:38:17,378 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/db8563b3627044808a0d5b4646b0a0c8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2c9291caf52b49cea3bca210d190920a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69e1107c71e349fca397ecb82fa7b7bd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/00f55f51bfc14f5899dab7df9f6d1daf, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6908ee6897f74e3d86c7b8a66c932109, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/ef13cfec09824fceb8eeb772bfca94bc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ae06d958be946129b88ffdba7b7a9b8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/e0ad8027ef09453a804f363bb69ff8c9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/9e9841dca1f84364902e65c9e7fff0a9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1a82204a75af44b69e849c815969987d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/98188398ef804fecb435417e80c5dd32, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6ef52441cf884d4489c065ccf56544a9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/15f72423d4144d4cb97d92462a69ebd2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/5fe94ff235434573a48029aca1cf0147, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/64d3fd6ea94245158e2b57e5069e438e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/4f37c1875ee940be8dfb68d8939f6f49, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b5e53dcb4d274024a58078f55f52871b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/8d242fd2109c440691268630f1c3b540, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1426902700d54fbb8e1374ec10fd42ee, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/77533df3946149a59537128b50ea9181, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b325f881e2a749eeb2351966fc0eeae0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/c525c2346e50410b90b6246059d1d085, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69c7e0f2600d4da2bab48ef313437b25] to archive 2024-11-13T18:38:17,379 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:17,380 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/db8563b3627044808a0d5b4646b0a0c8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/db8563b3627044808a0d5b4646b0a0c8 2024-11-13T18:38:17,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523157365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523157366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523157366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,382 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2c9291caf52b49cea3bca210d190920a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2c9291caf52b49cea3bca210d190920a 2024-11-13T18:38:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,384 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69e1107c71e349fca397ecb82fa7b7bd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69e1107c71e349fca397ecb82fa7b7bd 2024-11-13T18:38:17,392 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/00f55f51bfc14f5899dab7df9f6d1daf to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/00f55f51bfc14f5899dab7df9f6d1daf 2024-11-13T18:38:17,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6908ee6897f74e3d86c7b8a66c932109 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6908ee6897f74e3d86c7b8a66c932109 2024-11-13T18:38:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T18:38:17,394 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:17,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:17,395 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/ef13cfec09824fceb8eeb772bfca94bc to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/ef13cfec09824fceb8eeb772bfca94bc 2024-11-13T18:38:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-13T18:38:17,396 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ae06d958be946129b88ffdba7b7a9b8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ae06d958be946129b88ffdba7b7a9b8 2024-11-13T18:38:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:38:17,397 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:17,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/e0ad8027ef09453a804f363bb69ff8c9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/e0ad8027ef09453a804f363bb69ff8c9 2024-11-13T18:38:17,397 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:17,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:17,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/9e9841dca1f84364902e65c9e7fff0a9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/9e9841dca1f84364902e65c9e7fff0a9 2024-11-13T18:38:17,400 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1a82204a75af44b69e849c815969987d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1a82204a75af44b69e849c815969987d 2024-11-13T18:38:17,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/98188398ef804fecb435417e80c5dd32 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/98188398ef804fecb435417e80c5dd32 2024-11-13T18:38:17,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742250_1426 (size=39949) 2024-11-13T18:38:17,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6ef52441cf884d4489c065ccf56544a9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/6ef52441cf884d4489c065ccf56544a9 2024-11-13T18:38:17,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/15f72423d4144d4cb97d92462a69ebd2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/15f72423d4144d4cb97d92462a69ebd2 2024-11-13T18:38:17,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/5fe94ff235434573a48029aca1cf0147 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/5fe94ff235434573a48029aca1cf0147 2024-11-13T18:38:17,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/2795375abeba4a4ab8ed73e88f85438f 2024-11-13T18:38:17,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/64d3fd6ea94245158e2b57e5069e438e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/64d3fd6ea94245158e2b57e5069e438e 2024-11-13T18:38:17,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/4f37c1875ee940be8dfb68d8939f6f49 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/4f37c1875ee940be8dfb68d8939f6f49 2024-11-13T18:38:17,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b5e53dcb4d274024a58078f55f52871b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b5e53dcb4d274024a58078f55f52871b 2024-11-13T18:38:17,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/8d242fd2109c440691268630f1c3b540 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/8d242fd2109c440691268630f1c3b540 2024-11-13T18:38:17,412 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1426902700d54fbb8e1374ec10fd42ee to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/1426902700d54fbb8e1374ec10fd42ee 2024-11-13T18:38:17,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/65e95e30bb9849ae9af24fdfa741de89 is 50, key is test_row_0/B:col10/1731523096670/Put/seqid=0 2024-11-13T18:38:17,413 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/77533df3946149a59537128b50ea9181 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/77533df3946149a59537128b50ea9181 2024-11-13T18:38:17,414 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b325f881e2a749eeb2351966fc0eeae0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/b325f881e2a749eeb2351966fc0eeae0 2024-11-13T18:38:17,415 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/c525c2346e50410b90b6246059d1d085 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/c525c2346e50410b90b6246059d1d085 2024-11-13T18:38:17,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742251_1427 (size=12301) 2024-11-13T18:38:17,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69c7e0f2600d4da2bab48ef313437b25 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/69c7e0f2600d4da2bab48ef313437b25 2024-11-13T18:38:17,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/65e95e30bb9849ae9af24fdfa741de89 2024-11-13T18:38:17,422 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/27fd7329e9fc48c7861f10bbfdd349b4, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/cd9f612650a64a4abd8bb9802b023966, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/873c81e70afc4c09a91034f28f9fa6a1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/d84003405f4c4c6a9ef210e01bdd4058, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e909a477a5654687bab3683429c8c8b0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/dc71af65307041d7bda63e35f24e1d7f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/3360c7e3caba4fa6957e281382a85758, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/14049248143d48c19d174a4eac6c799c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad97c97ec12c4e0a8f03de0f3c18e28c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e4e33fecb57e4377819d3dc057e774c0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/da850c7774954c5ab7ca7ee8d87ec9e6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e2ae85213a9b47e18348eca893a04ee1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/495511d7ab3d447fb5dabe9199ceb964, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9a350eb0a82a4294a6cc987ba1c28374, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9dfc35289c8f46229692c42871390680, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e5dca9e898b3432ba81188107d899057, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/7ca6e332e4184714bc8a4a492b4c579f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/5b789fb2942c43dca05aba794dc3cda2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/c8cd20ab18d34a96aeb3a909a483b568, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2664cd6de6374513a7faf2e1d68d13f9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/69eb488e6b804611b673b5301f29ffad] to archive 2024-11-13T18:38:17,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:17,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/27fd7329e9fc48c7861f10bbfdd349b4 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/27fd7329e9fc48c7861f10bbfdd349b4 2024-11-13T18:38:17,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/cd9f612650a64a4abd8bb9802b023966 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/cd9f612650a64a4abd8bb9802b023966 2024-11-13T18:38:17,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/ad2da270d6a24a40bba8a7a7439a416b is 50, key is test_row_0/C:col10/1731523096670/Put/seqid=0 2024-11-13T18:38:17,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/873c81e70afc4c09a91034f28f9fa6a1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/873c81e70afc4c09a91034f28f9fa6a1 2024-11-13T18:38:17,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/d84003405f4c4c6a9ef210e01bdd4058 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/d84003405f4c4c6a9ef210e01bdd4058 2024-11-13T18:38:17,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742252_1428 (size=12301) 2024-11-13T18:38:17,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e909a477a5654687bab3683429c8c8b0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e909a477a5654687bab3683429c8c8b0 2024-11-13T18:38:17,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/dc71af65307041d7bda63e35f24e1d7f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/dc71af65307041d7bda63e35f24e1d7f 2024-11-13T18:38:17,443 INFO [Thread-1516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1516. 2024-11-13T18:38:17,443 DEBUG [Thread-1516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:17,443 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,443 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:17,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/3360c7e3caba4fa6957e281382a85758 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/3360c7e3caba4fa6957e281382a85758 2024-11-13T18:38:17,445 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/14049248143d48c19d174a4eac6c799c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/14049248143d48c19d174a4eac6c799c 2024-11-13T18:38:17,446 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad97c97ec12c4e0a8f03de0f3c18e28c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad97c97ec12c4e0a8f03de0f3c18e28c 2024-11-13T18:38:17,447 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e4e33fecb57e4377819d3dc057e774c0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e4e33fecb57e4377819d3dc057e774c0 2024-11-13T18:38:17,447 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/da850c7774954c5ab7ca7ee8d87ec9e6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/da850c7774954c5ab7ca7ee8d87ec9e6 2024-11-13T18:38:17,449 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e2ae85213a9b47e18348eca893a04ee1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e2ae85213a9b47e18348eca893a04ee1 2024-11-13T18:38:17,450 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/495511d7ab3d447fb5dabe9199ceb964 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/495511d7ab3d447fb5dabe9199ceb964 2024-11-13T18:38:17,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9a350eb0a82a4294a6cc987ba1c28374 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9a350eb0a82a4294a6cc987ba1c28374 2024-11-13T18:38:17,453 INFO [Thread-1508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1508. 2024-11-13T18:38:17,453 DEBUG [Thread-1508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:17,453 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,453 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:17,453 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9dfc35289c8f46229692c42871390680 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/9dfc35289c8f46229692c42871390680 2024-11-13T18:38:17,454 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c2c6326bdfc4e91a16ea6f0b5cedd7a 2024-11-13T18:38:17,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e5dca9e898b3432ba81188107d899057 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/e5dca9e898b3432ba81188107d899057 2024-11-13T18:38:17,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/7ca6e332e4184714bc8a4a492b4c579f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/7ca6e332e4184714bc8a4a492b4c579f 2024-11-13T18:38:17,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/5b789fb2942c43dca05aba794dc3cda2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/5b789fb2942c43dca05aba794dc3cda2 2024-11-13T18:38:17,457 INFO [Thread-1514 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1514. 2024-11-13T18:38:17,457 DEBUG [Thread-1514 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:17,457 INFO [Thread-1510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1510. 2024-11-13T18:38:17,457 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,457 DEBUG [Thread-1510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:17,457 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,457 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:17,458 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:17,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/c8cd20ab18d34a96aeb3a909a483b568 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/c8cd20ab18d34a96aeb3a909a483b568 2024-11-13T18:38:17,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2664cd6de6374513a7faf2e1d68d13f9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2664cd6de6374513a7faf2e1d68d13f9 2024-11-13T18:38:17,460 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/69eb488e6b804611b673b5301f29ffad to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/69eb488e6b804611b673b5301f29ffad 2024-11-13T18:38:17,462 INFO [Thread-1512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1512. 2024-11-13T18:38:17,462 DEBUG [Thread-1512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:17,463 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,463 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:17,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523157473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523157473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523157483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523157483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523157493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:38:17,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:38:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60794 deadline: 1731523157684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60780 deadline: 1731523157684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60748 deadline: 1731523157693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60774 deadline: 1731523157693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60816 deadline: 1731523157703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:17,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:38:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:38:17,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/ad2da270d6a24a40bba8a7a7439a416b 2024-11-13T18:38:17,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/2795375abeba4a4ab8ed73e88f85438f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2795375abeba4a4ab8ed73e88f85438f 2024-11-13T18:38:17,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2795375abeba4a4ab8ed73e88f85438f, entries=200, sequenceid=365, filesize=39.0 K 2024-11-13T18:38:17,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/65e95e30bb9849ae9af24fdfa741de89 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/65e95e30bb9849ae9af24fdfa741de89 2024-11-13T18:38:17,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/65e95e30bb9849ae9af24fdfa741de89, entries=150, sequenceid=365, filesize=12.0 K 2024-11-13T18:38:17,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/ad2da270d6a24a40bba8a7a7439a416b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad2da270d6a24a40bba8a7a7439a416b 2024-11-13T18:38:17,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad2da270d6a24a40bba8a7a7439a416b, entries=150, sequenceid=365, filesize=12.0 K 2024-11-13T18:38:17,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a70474ff27654302c682d5d28687cb3d in 514ms, sequenceid=365, compaction requested=true 2024-11-13T18:38:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a70474ff27654302c682d5d28687cb3d:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:17,843 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:17,843 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:17,844 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:17,844 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:17,844 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/A is initiating minor compaction (all files) 2024-11-13T18:38:17,844 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/B is initiating minor compaction (all files) 2024-11-13T18:38:17,844 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/A in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,844 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/B in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,844 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/597564033dda4c919c00383ffae290f7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/3c3afa24a55f4262a822860d4c0a8298, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2795375abeba4a4ab8ed73e88f85438f] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=100.8 K 2024-11-13T18:38:17,844 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,844 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/597564033dda4c919c00383ffae290f7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/3c3afa24a55f4262a822860d4c0a8298, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2795375abeba4a4ab8ed73e88f85438f] 2024-11-13T18:38:17,844 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/f8afa29f2e9142519158e90d620a4993, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ca32601fe584439bf3873e87d31b11d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/65e95e30bb9849ae9af24fdfa741de89] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=36.8 K 2024-11-13T18:38:17,844 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597564033dda4c919c00383ffae290f7, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731523093243 2024-11-13T18:38:17,844 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting f8afa29f2e9142519158e90d620a4993, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731523093243 2024-11-13T18:38:17,845 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c3afa24a55f4262a822860d4c0a8298, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731523095465 2024-11-13T18:38:17,845 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca32601fe584439bf3873e87d31b11d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731523095465 2024-11-13T18:38:17,845 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2795375abeba4a4ab8ed73e88f85438f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731523096667 2024-11-13T18:38:17,845 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e95e30bb9849ae9af24fdfa741de89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731523096667 2024-11-13T18:38:17,851 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:17,853 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:17,854 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/27fe12436253459fa6666833753b0502 is 50, key is test_row_0/B:col10/1731523096670/Put/seqid=0 2024-11-13T18:38:17,855 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411131175fd2ea696407f9cf3668743eb8dc1_a70474ff27654302c682d5d28687cb3d store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:17,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742253_1429 (size=12404) 2024-11-13T18:38:17,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T18:38:17,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:17,857 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411131175fd2ea696407f9cf3668743eb8dc1_a70474ff27654302c682d5d28687cb3d, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:17,858 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131175fd2ea696407f9cf3668743eb8dc1_a70474ff27654302c682d5d28687cb3d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:17,858 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T18:38:17,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742254_1430 (size=4469) 2024-11-13T18:38:17,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c187efbfa1584ab2bd155f7d99e86560_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_0/A:col10/1731523097364/Put/seqid=0 2024-11-13T18:38:17,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742255_1431 (size=12454) 2024-11-13T18:38:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:17,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. as already flushing 2024-11-13T18:38:17,995 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-11-13T18:38:17,995 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:17,995 INFO [Thread-1501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1501. 2024-11-13T18:38:17,995 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,995 DEBUG [Thread-1501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:17,995 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,995 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,995 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:17,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:17,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:18,005 INFO [Thread-1497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1497. 2024-11-13T18:38:18,005 INFO [Thread-1505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1505. 2024-11-13T18:38:18,005 DEBUG [Thread-1497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:18,005 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:18,005 DEBUG [Thread-1505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:18,005 DEBUG [Thread-1505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:18,005 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:18,005 DEBUG [Thread-1505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:18,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:18,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:18,014 INFO [Thread-1503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1503. 2024-11-13T18:38:18,014 DEBUG [Thread-1503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:18,014 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:18,014 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:18,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:38:18,262 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/27fe12436253459fa6666833753b0502 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/27fe12436253459fa6666833753b0502 2024-11-13T18:38:18,263 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#A#compaction#359 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:18,263 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/b59eeb863c1c41d3ad53b2338d5fcdc5 is 175, key is test_row_0/A:col10/1731523096670/Put/seqid=0 2024-11-13T18:38:18,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:18,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742256_1432 (size=31358) 2024-11-13T18:38:18,272 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c187efbfa1584ab2bd155f7d99e86560_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c187efbfa1584ab2bd155f7d99e86560_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:18,273 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/B of a70474ff27654302c682d5d28687cb3d into 27fe12436253459fa6666833753b0502(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:18,273 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:18,273 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/B, priority=13, startTime=1731523097843; duration=0sec 2024-11-13T18:38:18,273 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:18,273 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:B 2024-11-13T18:38:18,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/be045a1d248c4e4a812810b5249e21c6, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:18,273 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:18,274 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/b59eeb863c1c41d3ad53b2338d5fcdc5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/b59eeb863c1c41d3ad53b2338d5fcdc5 2024-11-13T18:38:18,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/be045a1d248c4e4a812810b5249e21c6 is 175, key is test_row_0/A:col10/1731523097364/Put/seqid=0 2024-11-13T18:38:18,274 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:18,274 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): a70474ff27654302c682d5d28687cb3d/C is initiating minor compaction (all files) 2024-11-13T18:38:18,274 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a70474ff27654302c682d5d28687cb3d/C in TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:18,274 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2cc78b381fef4fbb9ded05469f57dfb9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c212c6d9bcc43a28ba25f78a23442e9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad2da270d6a24a40bba8a7a7439a416b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp, totalSize=36.8 K 2024-11-13T18:38:18,274 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cc78b381fef4fbb9ded05469f57dfb9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731523093243 2024-11-13T18:38:18,275 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c212c6d9bcc43a28ba25f78a23442e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731523095465 2024-11-13T18:38:18,275 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2da270d6a24a40bba8a7a7439a416b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731523096667 2024-11-13T18:38:18,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742257_1433 (size=31255) 2024-11-13T18:38:18,280 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/A of a70474ff27654302c682d5d28687cb3d into b59eeb863c1c41d3ad53b2338d5fcdc5(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:18,280 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:18,280 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/A, priority=13, startTime=1731523097843; duration=0sec 2024-11-13T18:38:18,280 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:18,280 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:A 2024-11-13T18:38:18,282 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a70474ff27654302c682d5d28687cb3d#C#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:18,282 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/086494ea2b8642f3bbb656ef6bd75338 is 50, key is test_row_0/C:col10/1731523096670/Put/seqid=0 2024-11-13T18:38:18,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742258_1434 (size=12404) 2024-11-13T18:38:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:38:18,678 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/be045a1d248c4e4a812810b5249e21c6 2024-11-13T18:38:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/2d93060fc4d544ceb072a6d87763db3a is 50, key is test_row_0/B:col10/1731523097364/Put/seqid=0 2024-11-13T18:38:18,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742259_1435 (size=12301) 2024-11-13T18:38:18,692 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/086494ea2b8642f3bbb656ef6bd75338 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/086494ea2b8642f3bbb656ef6bd75338 2024-11-13T18:38:18,696 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a70474ff27654302c682d5d28687cb3d/C of a70474ff27654302c682d5d28687cb3d into 086494ea2b8642f3bbb656ef6bd75338(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:18,696 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:18,696 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., storeName=a70474ff27654302c682d5d28687cb3d/C, priority=13, startTime=1731523097843; duration=0sec 2024-11-13T18:38:18,696 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:18,696 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a70474ff27654302c682d5d28687cb3d:C 2024-11-13T18:38:19,087 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/2d93060fc4d544ceb072a6d87763db3a 2024-11-13T18:38:19,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/82a39058f50b48b0a4cd1ae34fcbe168 is 50, key is test_row_0/C:col10/1731523097364/Put/seqid=0 2024-11-13T18:38:19,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742260_1436 (size=12301) 2024-11-13T18:38:19,496 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/82a39058f50b48b0a4cd1ae34fcbe168 2024-11-13T18:38:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/be045a1d248c4e4a812810b5249e21c6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/be045a1d248c4e4a812810b5249e21c6 2024-11-13T18:38:19,504 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/be045a1d248c4e4a812810b5249e21c6, entries=150, sequenceid=380, filesize=30.5 K 2024-11-13T18:38:19,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/2d93060fc4d544ceb072a6d87763db3a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2d93060fc4d544ceb072a6d87763db3a 2024-11-13T18:38:19,512 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2d93060fc4d544ceb072a6d87763db3a, entries=150, sequenceid=380, filesize=12.0 K 2024-11-13T18:38:19,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/82a39058f50b48b0a4cd1ae34fcbe168 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/82a39058f50b48b0a4cd1ae34fcbe168 2024-11-13T18:38:19,516 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/82a39058f50b48b0a4cd1ae34fcbe168, entries=150, sequenceid=380, filesize=12.0 K 2024-11-13T18:38:19,517 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=33.54 KB/34350 for a70474ff27654302c682d5d28687cb3d in 1659ms, sequenceid=380, compaction requested=false 2024-11-13T18:38:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a70474ff27654302c682d5d28687cb3d: 2024-11-13T18:38:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-13T18:38:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-13T18:38:19,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T18:38:19,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-11-13T18:38:19,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1270 sec 2024-11-13T18:38:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T18:38:19,544 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 892 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2673 rows 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 889 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2664 rows 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 889 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2662 rows 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 891 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2667 rows 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 895 2024-11-13T18:38:19,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2681 rows 2024-11-13T18:38:19,544 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:38:19,544 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:38:19,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:19,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:19,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:19,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:38:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:38:19,548 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523099548"}]},"ts":"1731523099548"} 2024-11-13T18:38:19,549 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:38:19,549 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:38:19,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:38:19,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, UNASSIGN}] 2024-11-13T18:38:19,552 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, UNASSIGN 2024-11-13T18:38:19,553 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a70474ff27654302c682d5d28687cb3d, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:19,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, UNASSIGN because future has completed 2024-11-13T18:38:19,554 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:38:19,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:38:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:38:19,706 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:19,706 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:38:19,706 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing a70474ff27654302c682d5d28687cb3d, disabling compactions & flushes 2024-11-13T18:38:19,707 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:19,707 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:19,707 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. after waiting 0 ms 2024-11-13T18:38:19,707 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:19,707 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing a70474ff27654302c682d5d28687cb3d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T18:38:19,713 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e1e6172b3bbd473ab8e63fecb364c28e_a70474ff27654302c682d5d28687cb3d is 50, key is test_row_1/A:col10/1731523098004/Put/seqid=0 2024-11-13T18:38:19,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742261_1437 (size=9914) 2024-11-13T18:38:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:38:20,117 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:20,120 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e1e6172b3bbd473ab8e63fecb364c28e_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e1e6172b3bbd473ab8e63fecb364c28e_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:20,121 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/1f0c3473cb914334a916c415c07e84ec, store: [table=TestAcidGuarantees family=A region=a70474ff27654302c682d5d28687cb3d] 2024-11-13T18:38:20,122 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/1f0c3473cb914334a916c415c07e84ec is 175, key is test_row_1/A:col10/1731523098004/Put/seqid=0 2024-11-13T18:38:20,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742262_1438 (size=22561) 2024-11-13T18:38:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:38:20,525 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/1f0c3473cb914334a916c415c07e84ec 2024-11-13T18:38:20,536 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/a6ff1baad46942ab9837c622c6861a59 is 50, key is test_row_1/B:col10/1731523098004/Put/seqid=0 2024-11-13T18:38:20,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742263_1439 (size=9857) 2024-11-13T18:38:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:38:20,940 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/a6ff1baad46942ab9837c622c6861a59 2024-11-13T18:38:20,946 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/8b7468a6877943b1ae5bcce6d819b903 is 50, key is test_row_1/C:col10/1731523098004/Put/seqid=0 2024-11-13T18:38:20,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742264_1440 (size=9857) 2024-11-13T18:38:21,350 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/8b7468a6877943b1ae5bcce6d819b903 2024-11-13T18:38:21,355 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/A/1f0c3473cb914334a916c415c07e84ec as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1f0c3473cb914334a916c415c07e84ec 2024-11-13T18:38:21,358 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1f0c3473cb914334a916c415c07e84ec, entries=100, sequenceid=391, filesize=22.0 K 2024-11-13T18:38:21,359 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/B/a6ff1baad46942ab9837c622c6861a59 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/a6ff1baad46942ab9837c622c6861a59 2024-11-13T18:38:21,362 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/a6ff1baad46942ab9837c622c6861a59, entries=100, sequenceid=391, filesize=9.6 K 2024-11-13T18:38:21,363 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/.tmp/C/8b7468a6877943b1ae5bcce6d819b903 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/8b7468a6877943b1ae5bcce6d819b903 2024-11-13T18:38:21,365 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/8b7468a6877943b1ae5bcce6d819b903, entries=100, sequenceid=391, filesize=9.6 K 2024-11-13T18:38:21,366 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a70474ff27654302c682d5d28687cb3d in 1659ms, sequenceid=391, compaction requested=true 2024-11-13T18:38:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/597564033dda4c919c00383ffae290f7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/3c3afa24a55f4262a822860d4c0a8298, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2795375abeba4a4ab8ed73e88f85438f] to archive 2024-11-13T18:38:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:21,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/597564033dda4c919c00383ffae290f7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/597564033dda4c919c00383ffae290f7 2024-11-13T18:38:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/3c3afa24a55f4262a822860d4c0a8298 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/3c3afa24a55f4262a822860d4c0a8298 2024-11-13T18:38:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2795375abeba4a4ab8ed73e88f85438f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/2795375abeba4a4ab8ed73e88f85438f 2024-11-13T18:38:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/f8afa29f2e9142519158e90d620a4993, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ca32601fe584439bf3873e87d31b11d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/65e95e30bb9849ae9af24fdfa741de89] to archive 2024-11-13T18:38:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/f8afa29f2e9142519158e90d620a4993 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/f8afa29f2e9142519158e90d620a4993 2024-11-13T18:38:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ca32601fe584439bf3873e87d31b11d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/3ca32601fe584439bf3873e87d31b11d 2024-11-13T18:38:21,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/65e95e30bb9849ae9af24fdfa741de89 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/65e95e30bb9849ae9af24fdfa741de89 2024-11-13T18:38:21,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2cc78b381fef4fbb9ded05469f57dfb9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c212c6d9bcc43a28ba25f78a23442e9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad2da270d6a24a40bba8a7a7439a416b] to archive 2024-11-13T18:38:21,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:21,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2cc78b381fef4fbb9ded05469f57dfb9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/2cc78b381fef4fbb9ded05469f57dfb9 2024-11-13T18:38:21,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c212c6d9bcc43a28ba25f78a23442e9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/6c212c6d9bcc43a28ba25f78a23442e9 2024-11-13T18:38:21,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad2da270d6a24a40bba8a7a7439a416b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/ad2da270d6a24a40bba8a7a7439a416b 2024-11-13T18:38:21,384 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/recovered.edits/394.seqid, newMaxSeqId=394, maxSeqId=4 2024-11-13T18:38:21,384 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. 2024-11-13T18:38:21,384 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for a70474ff27654302c682d5d28687cb3d: Waiting for close lock at 1731523099706Running coprocessor pre-close hooks at 1731523099706Disabling compacts and flushes for region at 1731523099706Disabling writes for close at 1731523099707 (+1 ms)Obtaining lock to block concurrent updates at 1731523099707Preparing flush snapshotting stores in a70474ff27654302c682d5d28687cb3d at 1731523099707Finished memstore snapshotting TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731523099707Flushing stores of TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d. at 1731523099708 (+1 ms)Flushing a70474ff27654302c682d5d28687cb3d/A: creating writer at 1731523099708Flushing mob file a70474ff27654302c682d5d28687cb3d/A: appending metadata at 1731523099713 (+5 ms)Flushing mob file a70474ff27654302c682d5d28687cb3d/A: closing flushed file at 1731523099713Flushing a70474ff27654302c682d5d28687cb3d/A: appending metadata at 1731523100121 (+408 ms)Flushing a70474ff27654302c682d5d28687cb3d/A: closing flushed file at 1731523100121Flushing a70474ff27654302c682d5d28687cb3d/B: creating writer at 1731523100530 (+409 ms)Flushing a70474ff27654302c682d5d28687cb3d/B: appending metadata at 1731523100535 (+5 ms)Flushing a70474ff27654302c682d5d28687cb3d/B: closing flushed file at 1731523100535Flushing a70474ff27654302c682d5d28687cb3d/C: creating writer at 1731523100943 (+408 ms)Flushing a70474ff27654302c682d5d28687cb3d/C: appending metadata at 1731523100945 (+2 ms)Flushing a70474ff27654302c682d5d28687cb3d/C: closing flushed file at 1731523100945Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cbd1c15: reopening flushed file at 1731523101354 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bd3cf44: reopening flushed file at 1731523101359 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c7b5ee: reopening flushed file at 1731523101362 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a70474ff27654302c682d5d28687cb3d in 1659ms, sequenceid=391, compaction requested=true at 1731523101366 (+4 ms)Writing region close event to WAL at 1731523101381 (+15 ms)Running coprocessor post-close hooks at 1731523101384 (+3 ms)Closed at 1731523101384 2024-11-13T18:38:21,386 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,387 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a70474ff27654302c682d5d28687cb3d, regionState=CLOSED 2024-11-13T18:38:21,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:38:21,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T18:38:21,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure a70474ff27654302c682d5d28687cb3d, server=39e84130bbc9,35235,1731522975587 in 1.8350 sec 2024-11-13T18:38:21,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-13T18:38:21,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a70474ff27654302c682d5d28687cb3d, UNASSIGN in 1.8390 sec 2024-11-13T18:38:21,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T18:38:21,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8420 sec 2024-11-13T18:38:21,395 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523101395"}]},"ts":"1731523101395"} 2024-11-13T18:38:21,396 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:38:21,396 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:38:21,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8510 sec 2024-11-13T18:38:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T18:38:21,694 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:21,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:38:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:21,696 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T18:38:21,701 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:21,703 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,706 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/recovered.edits] 2024-11-13T18:38:21,710 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1f0c3473cb914334a916c415c07e84ec to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/1f0c3473cb914334a916c415c07e84ec 2024-11-13T18:38:21,711 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/b59eeb863c1c41d3ad53b2338d5fcdc5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/b59eeb863c1c41d3ad53b2338d5fcdc5 2024-11-13T18:38:21,713 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/be045a1d248c4e4a812810b5249e21c6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/A/be045a1d248c4e4a812810b5249e21c6 2024-11-13T18:38:21,717 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/27fe12436253459fa6666833753b0502 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/27fe12436253459fa6666833753b0502 2024-11-13T18:38:21,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2d93060fc4d544ceb072a6d87763db3a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/2d93060fc4d544ceb072a6d87763db3a 2024-11-13T18:38:21,720 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/a6ff1baad46942ab9837c622c6861a59 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/B/a6ff1baad46942ab9837c622c6861a59 2024-11-13T18:38:21,722 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/086494ea2b8642f3bbb656ef6bd75338 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/086494ea2b8642f3bbb656ef6bd75338 2024-11-13T18:38:21,724 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/82a39058f50b48b0a4cd1ae34fcbe168 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/82a39058f50b48b0a4cd1ae34fcbe168 2024-11-13T18:38:21,725 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/8b7468a6877943b1ae5bcce6d819b903 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/C/8b7468a6877943b1ae5bcce6d819b903 2024-11-13T18:38:21,729 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/recovered.edits/394.seqid to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d/recovered.edits/394.seqid 2024-11-13T18:38:21,730 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,730 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:38:21,730 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:38:21,731 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:38:21,734 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302478100beb6493686888e32582f3167_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111302478100beb6493686888e32582f3167_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,735 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111314ee6378d1324cf6bb12f42157da5161_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111314ee6378d1324cf6bb12f42157da5161_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,736 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111318baf14c549a45d4af2a9950d4ee1278_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111318baf14c549a45d4af2a9950d4ee1278_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,738 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135e9ed44a1ba6484c9e5f910f0215e95f_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135e9ed44a1ba6484c9e5f910f0215e95f_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,739 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113713084b215464812bd5bb6a20b627f08_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113713084b215464812bd5bb6a20b627f08_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,740 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137414aef9ddd440e18e26767f34385e3f_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137414aef9ddd440e18e26767f34385e3f_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,741 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b91738b9a544dc6875f89590fd35f9d_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b91738b9a544dc6875f89590fd35f9d_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,742 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113865a428f721d4be7bb6a4e5674105691_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113865a428f721d4be7bb6a4e5674105691_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,744 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113967c869684bc46349ac46238758d8bc2_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113967c869684bc46349ac46238758d8bc2_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,745 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a2d91566028948a1a15fafe9ce9e95da_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a2d91566028948a1a15fafe9ce9e95da_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,746 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ad53757f15ff4709b2daa875627a98d1_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ad53757f15ff4709b2daa875627a98d1_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,747 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1ba4452e799477ebeb576acefdd8ec2_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1ba4452e799477ebeb576acefdd8ec2_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,749 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd13eb34b390483191dfb97d24a67cb7_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bd13eb34b390483191dfb97d24a67cb7_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,751 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bf170dce92a24aeba4dba1208587b922_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bf170dce92a24aeba4dba1208587b922_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,752 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c187efbfa1584ab2bd155f7d99e86560_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c187efbfa1584ab2bd155f7d99e86560_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,757 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c9d11d09f8124c82ad4ad5f678c3d0cd_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c9d11d09f8124c82ad4ad5f678c3d0cd_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,760 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cfc45ac918e24f308cc459218830b95f_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cfc45ac918e24f308cc459218830b95f_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,761 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d17cf084152f4259857f0d60c9e910c0_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d17cf084152f4259857f0d60c9e910c0_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,763 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d7eb5a507bdc486a9d9cfe05fec5b7fa_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d7eb5a507bdc486a9d9cfe05fec5b7fa_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,764 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc1f51330be144a68ff2a73bfa345c8a_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc1f51330be144a68ff2a73bfa345c8a_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,765 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e1e6172b3bbd473ab8e63fecb364c28e_a70474ff27654302c682d5d28687cb3d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e1e6172b3bbd473ab8e63fecb364c28e_a70474ff27654302c682d5d28687cb3d 2024-11-13T18:38:21,766 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:38:21,768 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:21,771 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:38:21,773 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:38:21,774 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:21,774 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:38:21,774 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731523101774"}]},"ts":"9223372036854775807"} 2024-11-13T18:38:21,776 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:38:21,776 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a70474ff27654302c682d5d28687cb3d, NAME => 'TestAcidGuarantees,,1731523075300.a70474ff27654302c682d5d28687cb3d.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:38:21,776 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:38:21,777 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731523101776"}]},"ts":"9223372036854775807"} 2024-11-13T18:38:21,778 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:38:21,779 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:21,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-11-13T18:38:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T18:38:21,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:38:21,804 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:21,815 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 236) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1355 (was 1804), ProcessCount=11 (was 11), AvailableMemoryMB=7878 (was 8352) 2024-11-13T18:38:21,825 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1355, ProcessCount=11, AvailableMemoryMB=7878 2024-11-13T18:38:21,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:38:21,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:38:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:21,828 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:38:21,829 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:21,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-13T18:38:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-13T18:38:21,830 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:38:21,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742265_1441 (size=959) 2024-11-13T18:38:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-13T18:38:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-13T18:38:22,237 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 2024-11-13T18:38:22,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742266_1442 (size=53) 2024-11-13T18:38:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-13T18:38:22,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:38:22,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, disabling compactions & flushes 2024-11-13T18:38:22,643 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:22,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:22,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. after waiting 0 ms 2024-11-13T18:38:22,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:22,644 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:22,644 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: Waiting for close lock at 1731523102643Disabling compacts and flushes for region at 1731523102643Disabling writes for close at 1731523102643Writing region close event to WAL at 1731523102643Closed at 1731523102643 2024-11-13T18:38:22,645 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:38:22,645 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731523102645"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731523102645"}]},"ts":"1731523102645"} 2024-11-13T18:38:22,646 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:38:22,647 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:38:22,647 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523102647"}]},"ts":"1731523102647"} 2024-11-13T18:38:22,648 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:38:22,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, ASSIGN}] 2024-11-13T18:38:22,650 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, ASSIGN 2024-11-13T18:38:22,650 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, ASSIGN; state=OFFLINE, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=false 2024-11-13T18:38:22,801 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:22,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, ASSIGN because future has completed 2024-11-13T18:38:22,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:38:22,958 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:22,958 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:38:22,959 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,960 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:38:22,960 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,960 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,961 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,962 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 columnFamilyName A 2024-11-13T18:38:22,962 DEBUG [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:22,962 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] regionserver.HStore(327): Store=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:22,962 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-13T18:38:22,963 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 columnFamilyName B 2024-11-13T18:38:22,963 DEBUG [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:22,963 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] regionserver.HStore(327): Store=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:22,963 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,964 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 columnFamilyName C 2024-11-13T18:38:22,964 DEBUG [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:22,965 INFO [StoreOpener-1e1cbe3c8a3f4b52fb2c2bfe3ffb9166-1 {}] regionserver.HStore(327): Store=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:22,965 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,965 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,965 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,966 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,966 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,966 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:38:22,967 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,969 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:38:22,969 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64269465, jitterRate=-0.04231034219264984}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:38:22,969 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:22,969 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: Running coprocessor pre-open hook at 1731523102960Writing region info on filesystem at 1731523102960Initializing all the Stores at 1731523102960Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523102960Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523102960Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523102960Cleaning up temporary data from old regions at 1731523102966 (+6 ms)Running coprocessor post-open hooks at 1731523102969 (+3 ms)Region opened successfully at 1731523102969 2024-11-13T18:38:22,970 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., pid=114, masterSystemTime=1731523102956 2024-11-13T18:38:22,972 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:22,972 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:22,973 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:22,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:38:22,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T18:38:22,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 in 172 msec 2024-11-13T18:38:22,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-13T18:38:22,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, ASSIGN in 327 msec 2024-11-13T18:38:22,979 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:38:22,979 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523102979"}]},"ts":"1731523102979"} 2024-11-13T18:38:22,980 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:38:22,980 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:38:22,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-13T18:38:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-13T18:38:23,973 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:23,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:23,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:23,975 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ab5e17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:23,977 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6dd199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,978 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:23,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:23,981 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:38:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 39e84130bbc9,36105,1731522974847 2024-11-13T18:38:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b66609f 2024-11-13T18:38:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:38:23,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:38:23,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:23,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:23,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf4e0a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:23,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d3e8cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:23,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:23,987 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2779e32c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:23,988 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56d6f01f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:23,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:23,990 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a07d5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:23,991 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a1c8a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:23,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:23,992 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dda410a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:23,993 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2f4785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:23,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:23,995 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68323681, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:23,996 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24fa861a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:23,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:23,998 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c6755e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:23,999 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393c2963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:24,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:24,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689b4ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:24,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a13343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:24,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:24,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47218634, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:24,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c48119, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:24,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:24,008 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3867eef5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:24,009 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3561543d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c1b90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:24,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:24,011 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347824af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:24,012 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed0337, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:24,013 DEBUG [Thread-2002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,013 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,013 DEBUG [Thread-1996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,014 DEBUG [Thread-2011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,015 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:24,015 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,015 DEBUG [Thread-2009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,016 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-13T18:38:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,016 DEBUG [Thread-1998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,017 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,018 DEBUG [Thread-2007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:24,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:24,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:38:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,028 DEBUG [Thread-2013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:24,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2] 2024-11-13T18:38:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:24,032 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:24,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:38:24,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/f34b98d46c7041ed908c18b08a3594d3 is 50, key is test_row_0/A:col10/1731523104030/Put/seqid=0 2024-11-13T18:38:24,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742267_1443 (size=16681) 2024-11-13T18:38:24,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/f34b98d46c7041ed908c18b08a3594d3 2024-11-13T18:38:24,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/417e9a6a1dfd4a84bab31a44f4a44dc3 is 50, key is test_row_0/B:col10/1731523104030/Put/seqid=0 2024-11-13T18:38:24,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742268_1444 (size=12001) 2024-11-13T18:38:24,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/417e9a6a1dfd4a84bab31a44f4a44dc3 2024-11-13T18:38:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:38:24,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/67ff827c56544a28a1511fbd7c69ce28 is 50, key is test_row_0/C:col10/1731523104030/Put/seqid=0 2024-11-13T18:38:24,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742269_1445 (size=12001) 2024-11-13T18:38:24,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523164120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523164121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523164127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523164137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523164138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:38:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:24,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523164243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523164243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523164243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523164263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523164263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:38:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:24,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:38:24,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523164453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523164464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523164464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:38:24,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:24,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:24,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:24,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523164483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523164483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/67ff827c56544a28a1511fbd7c69ce28 2024-11-13T18:38:24,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/f34b98d46c7041ed908c18b08a3594d3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/f34b98d46c7041ed908c18b08a3594d3 2024-11-13T18:38:24,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/f34b98d46c7041ed908c18b08a3594d3, entries=250, sequenceid=13, filesize=16.3 K 2024-11-13T18:38:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/417e9a6a1dfd4a84bab31a44f4a44dc3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/417e9a6a1dfd4a84bab31a44f4a44dc3 2024-11-13T18:38:24,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/417e9a6a1dfd4a84bab31a44f4a44dc3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:38:24,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/67ff827c56544a28a1511fbd7c69ce28 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/67ff827c56544a28a1511fbd7c69ce28 2024-11-13T18:38:24,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/67ff827c56544a28a1511fbd7c69ce28, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T18:38:24,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 519ms, sequenceid=13, compaction requested=false 2024-11-13T18:38:24,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:24,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T18:38:24,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:24,633 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:38:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/6c060f6b1e854a85bebfa054f07ba7be is 50, key is test_row_0/A:col10/1731523104119/Put/seqid=0 2024-11-13T18:38:24,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742270_1446 (size=12001) 2024-11-13T18:38:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:38:24,645 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/6c060f6b1e854a85bebfa054f07ba7be 2024-11-13T18:38:24,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/c40cd6d049344890bc29f01834934684 is 50, key is test_row_0/B:col10/1731523104119/Put/seqid=0 2024-11-13T18:38:24,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742271_1447 (size=12001) 2024-11-13T18:38:24,662 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/c40cd6d049344890bc29f01834934684 2024-11-13T18:38:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/095aa0b588d1482abf7e5978ecd41d77 is 50, key is test_row_0/C:col10/1731523104119/Put/seqid=0 2024-11-13T18:38:24,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742272_1448 (size=12001) 2024-11-13T18:38:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:24,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523164802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523164803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523164803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523164804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523164804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523164923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523164924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523164924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,078 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/095aa0b588d1482abf7e5978ecd41d77 2024-11-13T18:38:25,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/6c060f6b1e854a85bebfa054f07ba7be as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6c060f6b1e854a85bebfa054f07ba7be 2024-11-13T18:38:25,086 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6c060f6b1e854a85bebfa054f07ba7be, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T18:38:25,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/c40cd6d049344890bc29f01834934684 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c40cd6d049344890bc29f01834934684 2024-11-13T18:38:25,094 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c40cd6d049344890bc29f01834934684, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T18:38:25,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/095aa0b588d1482abf7e5978ecd41d77 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/095aa0b588d1482abf7e5978ecd41d77 2024-11-13T18:38:25,098 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/095aa0b588d1482abf7e5978ecd41d77, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T18:38:25,099 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 466ms, sequenceid=38, compaction requested=false 2024-11-13T18:38:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-13T18:38:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-13T18:38:25,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-13T18:38:25,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0820 sec 2024-11-13T18:38:25,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0890 sec 2024-11-13T18:38:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:25,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:25,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:38:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T18:38:25,153 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:25,154 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:25,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/a111a6d381e74e26b012143115d75020 is 50, key is test_row_0/A:col10/1731523105148/Put/seqid=0 2024-11-13T18:38:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-13T18:38:25,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:38:25,156 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:25,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:25,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742273_1449 (size=16681) 2024-11-13T18:38:25,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/a111a6d381e74e26b012143115d75020 2024-11-13T18:38:25,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/d193baa63aac41fcb043d56e21e409ce is 50, key is test_row_0/B:col10/1731523105148/Put/seqid=0 2024-11-13T18:38:25,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742274_1450 (size=12001) 2024-11-13T18:38:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:38:25,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523165255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523165259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523165271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:38:25,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:25,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523165323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523165323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523165373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523165383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523165383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:38:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:38:25,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/d193baa63aac41fcb043d56e21e409ce 2024-11-13T18:38:25,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ad2be57fa2f941ddbb72f453350f96eb is 50, key is test_row_0/C:col10/1731523105148/Put/seqid=0 2024-11-13T18:38:25,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742275_1451 (size=12001) 2024-11-13T18:38:25,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523165593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523165593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523165593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:38:25,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:25,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:38:25,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:25,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:38:25,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:38:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:25,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:25,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523165913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523165914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523165914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:25,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ad2be57fa2f941ddbb72f453350f96eb 2024-11-13T18:38:25,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/a111a6d381e74e26b012143115d75020 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a111a6d381e74e26b012143115d75020 2024-11-13T18:38:25,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a111a6d381e74e26b012143115d75020, entries=250, sequenceid=51, filesize=16.3 K 2024-11-13T18:38:25,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/d193baa63aac41fcb043d56e21e409ce as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/d193baa63aac41fcb043d56e21e409ce 2024-11-13T18:38:25,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/d193baa63aac41fcb043d56e21e409ce, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T18:38:25,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ad2be57fa2f941ddbb72f453350f96eb as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ad2be57fa2f941ddbb72f453350f96eb 2024-11-13T18:38:26,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ad2be57fa2f941ddbb72f453350f96eb, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T18:38:26,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 852ms, sequenceid=51, compaction requested=true 2024-11-13T18:38:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:26,001 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:26,001 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:26,001 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:26,001 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A is initiating minor compaction (all files) 2024-11-13T18:38:26,002 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:26,002 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/f34b98d46c7041ed908c18b08a3594d3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6c060f6b1e854a85bebfa054f07ba7be, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a111a6d381e74e26b012143115d75020] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=44.3 K 2024-11-13T18:38:26,002 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:26,002 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B is initiating minor compaction (all files) 2024-11-13T18:38:26,002 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f34b98d46c7041ed908c18b08a3594d3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731523104022 2024-11-13T18:38:26,002 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:26,002 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/417e9a6a1dfd4a84bab31a44f4a44dc3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c40cd6d049344890bc29f01834934684, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/d193baa63aac41fcb043d56e21e409ce] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=35.2 K 2024-11-13T18:38:26,002 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c060f6b1e854a85bebfa054f07ba7be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731523104119 2024-11-13T18:38:26,002 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 417e9a6a1dfd4a84bab31a44f4a44dc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731523104022 2024-11-13T18:38:26,003 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a111a6d381e74e26b012143115d75020, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731523104800 2024-11-13T18:38:26,003 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c40cd6d049344890bc29f01834934684, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731523104119 2024-11-13T18:38:26,003 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting d193baa63aac41fcb043d56e21e409ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731523104800 2024-11-13T18:38:26,014 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:26,014 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:26,014 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/e4a407508b9a4986b508da9ed26c6e45 is 50, key is test_row_0/B:col10/1731523105148/Put/seqid=0 2024-11-13T18:38:26,015 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/cc3222dbd5ab49249ed35bd274f6b1b3 is 50, key is test_row_0/A:col10/1731523105148/Put/seqid=0 2024-11-13T18:38:26,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742276_1452 (size=12104) 2024-11-13T18:38:26,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742277_1453 (size=12104) 2024-11-13T18:38:26,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T18:38:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:26,077 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T18:38:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ea6e3ee265954d35a44b1e8a2b44bdb7 is 50, key is test_row_0/A:col10/1731523105269/Put/seqid=0 2024-11-13T18:38:26,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742278_1454 (size=12001) 2024-11-13T18:38:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:38:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:26,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:26,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523166396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523166397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,422 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/cc3222dbd5ab49249ed35bd274f6b1b3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cc3222dbd5ab49249ed35bd274f6b1b3 2024-11-13T18:38:26,425 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/e4a407508b9a4986b508da9ed26c6e45 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e4a407508b9a4986b508da9ed26c6e45 2024-11-13T18:38:26,426 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into cc3222dbd5ab49249ed35bd274f6b1b3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:26,426 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:26,426 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, priority=13, startTime=1731523106000; duration=0sec 2024-11-13T18:38:26,426 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:26,426 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A 2024-11-13T18:38:26,426 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:26,429 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:26,429 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C is initiating minor compaction (all files) 2024-11-13T18:38:26,429 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:26,430 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/67ff827c56544a28a1511fbd7c69ce28, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/095aa0b588d1482abf7e5978ecd41d77, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ad2be57fa2f941ddbb72f453350f96eb] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=35.2 K 2024-11-13T18:38:26,430 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ff827c56544a28a1511fbd7c69ce28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731523104022 2024-11-13T18:38:26,431 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 095aa0b588d1482abf7e5978ecd41d77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731523104119 2024-11-13T18:38:26,431 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad2be57fa2f941ddbb72f453350f96eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731523104800 2024-11-13T18:38:26,432 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into e4a407508b9a4986b508da9ed26c6e45(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:26,432 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:26,432 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, priority=13, startTime=1731523106001; duration=0sec 2024-11-13T18:38:26,432 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:26,432 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B 2024-11-13T18:38:26,438 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:26,438 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/0cfe3a9dcfc74a828a7cd63b9684b0ec is 50, key is test_row_0/C:col10/1731523105148/Put/seqid=0 2024-11-13T18:38:26,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742279_1455 (size=12104) 2024-11-13T18:38:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523166433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523166433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523166433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,486 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ea6e3ee265954d35a44b1e8a2b44bdb7 2024-11-13T18:38:26,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/a11ab326a97044c78131e2bb82cb6179 is 50, key is test_row_0/B:col10/1731523105269/Put/seqid=0 2024-11-13T18:38:26,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742280_1456 (size=12001) 2024-11-13T18:38:26,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523166514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523166523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523166733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523166734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:26,848 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/0cfe3a9dcfc74a828a7cd63b9684b0ec as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0cfe3a9dcfc74a828a7cd63b9684b0ec 2024-11-13T18:38:26,852 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 0cfe3a9dcfc74a828a7cd63b9684b0ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:26,852 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:26,852 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, priority=13, startTime=1731523106001; duration=0sec 2024-11-13T18:38:26,852 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:26,852 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C 2024-11-13T18:38:26,900 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:38:26,905 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/a11ab326a97044c78131e2bb82cb6179 2024-11-13T18:38:26,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a878637ab6c54af89455cd65b61fdc6f is 50, key is test_row_0/C:col10/1731523105269/Put/seqid=0 2024-11-13T18:38:26,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742281_1457 (size=12001) 2024-11-13T18:38:26,928 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a878637ab6c54af89455cd65b61fdc6f 2024-11-13T18:38:26,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ea6e3ee265954d35a44b1e8a2b44bdb7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea6e3ee265954d35a44b1e8a2b44bdb7 2024-11-13T18:38:26,935 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea6e3ee265954d35a44b1e8a2b44bdb7, entries=150, sequenceid=74, filesize=11.7 K 2024-11-13T18:38:26,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/a11ab326a97044c78131e2bb82cb6179 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a11ab326a97044c78131e2bb82cb6179 2024-11-13T18:38:26,946 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a11ab326a97044c78131e2bb82cb6179, entries=150, sequenceid=74, filesize=11.7 K 2024-11-13T18:38:26,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a878637ab6c54af89455cd65b61fdc6f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a878637ab6c54af89455cd65b61fdc6f 2024-11-13T18:38:26,951 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a878637ab6c54af89455cd65b61fdc6f, entries=150, sequenceid=74, filesize=11.7 K 2024-11-13T18:38:26,952 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 876ms, sequenceid=74, compaction requested=false 2024-11-13T18:38:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-13T18:38:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-13T18:38:26,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-13T18:38:26,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-13T18:38:26,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8040 sec 2024-11-13T18:38:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:27,057 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:27,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T18:38:27,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/00218c3f044f4d3e95a7c7081be6a949 is 50, key is test_row_0/A:col10/1731523106394/Put/seqid=0 2024-11-13T18:38:27,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742282_1458 (size=12001) 2024-11-13T18:38:27,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523167176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523167176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523167283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523167283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T18:38:27,304 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:27,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-13T18:38:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:38:27,305 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:27,306 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:27,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:38:27,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:27,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:27,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523167453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,463 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/00218c3f044f4d3e95a7c7081be6a949 2024-11-13T18:38:27,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/c54eddebf2094229911cdd338df552cd is 50, key is test_row_0/B:col10/1731523106394/Put/seqid=0 2024-11-13T18:38:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523167463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523167463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,476 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,477 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742283_1459 (size=12001) 2024-11-13T18:38:27,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523167504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523167504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:27,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:27,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:38:27,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:27,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:27,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523167833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523167833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:27,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/c54eddebf2094229911cdd338df552cd 2024-11-13T18:38:27,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/84e20aeae908429ab85b3e7ed608d93c is 50, key is test_row_0/C:col10/1731523106394/Put/seqid=0 2024-11-13T18:38:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742284_1460 (size=12001) 2024-11-13T18:38:27,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:27,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:27,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:38:28,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:28,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/84e20aeae908429ab85b3e7ed608d93c 2024-11-13T18:38:28,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/00218c3f044f4d3e95a7c7081be6a949 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/00218c3f044f4d3e95a7c7081be6a949 2024-11-13T18:38:28,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/00218c3f044f4d3e95a7c7081be6a949, entries=150, sequenceid=92, filesize=11.7 K 2024-11-13T18:38:28,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/c54eddebf2094229911cdd338df552cd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c54eddebf2094229911cdd338df552cd 2024-11-13T18:38:28,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c54eddebf2094229911cdd338df552cd, entries=150, sequenceid=92, filesize=11.7 K 2024-11-13T18:38:28,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/84e20aeae908429ab85b3e7ed608d93c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/84e20aeae908429ab85b3e7ed608d93c 2024-11-13T18:38:28,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/84e20aeae908429ab85b3e7ed608d93c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-13T18:38:28,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 1250ms, sequenceid=92, compaction requested=true 2024-11-13T18:38:28,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:28,307 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:28,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:28,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:28,308 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:28,308 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:28,308 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A is initiating minor compaction (all files) 2024-11-13T18:38:28,309 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,309 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cc3222dbd5ab49249ed35bd274f6b1b3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea6e3ee265954d35a44b1e8a2b44bdb7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/00218c3f044f4d3e95a7c7081be6a949] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=35.3 K 2024-11-13T18:38:28,309 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc3222dbd5ab49249ed35bd274f6b1b3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731523104800 2024-11-13T18:38:28,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:28,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:28,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:28,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:28,309 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:28,309 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea6e3ee265954d35a44b1e8a2b44bdb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731523105235 2024-11-13T18:38:28,309 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B is initiating minor compaction (all files) 2024-11-13T18:38:28,309 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,309 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e4a407508b9a4986b508da9ed26c6e45, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a11ab326a97044c78131e2bb82cb6179, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c54eddebf2094229911cdd338df552cd] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=35.3 K 2024-11-13T18:38:28,309 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00218c3f044f4d3e95a7c7081be6a949, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523106394 2024-11-13T18:38:28,310 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e4a407508b9a4986b508da9ed26c6e45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731523104800 2024-11-13T18:38:28,310 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a11ab326a97044c78131e2bb82cb6179, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731523105235 2024-11-13T18:38:28,310 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c54eddebf2094229911cdd338df552cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523106394 2024-11-13T18:38:28,320 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#A#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:28,320 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/7c8b193c26ff4e669ef33d632ba39b13 is 50, key is test_row_0/A:col10/1731523106394/Put/seqid=0 2024-11-13T18:38:28,322 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#B#compaction#387 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:28,323 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/a8da9a759a834755a92c2a5659a78913 is 50, key is test_row_0/B:col10/1731523106394/Put/seqid=0 2024-11-13T18:38:28,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742285_1461 (size=12207) 2024-11-13T18:38:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742286_1462 (size=12207) 2024-11-13T18:38:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:28,353 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:28,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T18:38:28,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/cf220dcbe24a45a9b11a0736ef0acb72 is 50, key is test_row_0/A:col10/1731523107174/Put/seqid=0 2024-11-13T18:38:28,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742287_1463 (size=14341) 2024-11-13T18:38:28,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/cf220dcbe24a45a9b11a0736ef0acb72 2024-11-13T18:38:28,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:28,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:28,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/58eff25a72af4129956142a7920f302c is 50, key is test_row_0/B:col10/1731523107174/Put/seqid=0 2024-11-13T18:38:28,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742288_1464 (size=12001) 2024-11-13T18:38:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523168432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:38:28,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523168434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:28,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523168543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:28,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523168554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:28,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,739 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/a8da9a759a834755a92c2a5659a78913 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a8da9a759a834755a92c2a5659a78913 2024-11-13T18:38:28,739 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/7c8b193c26ff4e669ef33d632ba39b13 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7c8b193c26ff4e669ef33d632ba39b13 2024-11-13T18:38:28,745 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 7c8b193c26ff4e669ef33d632ba39b13(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:28,746 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:28,746 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, priority=13, startTime=1731523108307; duration=0sec 2024-11-13T18:38:28,746 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:28,746 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A 2024-11-13T18:38:28,746 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:28,746 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into a8da9a759a834755a92c2a5659a78913(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:28,746 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:28,746 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, priority=13, startTime=1731523108308; duration=0sec 2024-11-13T18:38:28,746 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:28,746 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B 2024-11-13T18:38:28,746 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:28,747 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C is initiating minor compaction (all files) 2024-11-13T18:38:28,747 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,747 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0cfe3a9dcfc74a828a7cd63b9684b0ec, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a878637ab6c54af89455cd65b61fdc6f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/84e20aeae908429ab85b3e7ed608d93c] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=35.3 K 2024-11-13T18:38:28,747 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cfe3a9dcfc74a828a7cd63b9684b0ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731523104800 2024-11-13T18:38:28,747 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a878637ab6c54af89455cd65b61fdc6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731523105235 2024-11-13T18:38:28,747 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e20aeae908429ab85b3e7ed608d93c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523106394 2024-11-13T18:38:28,754 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:28,755 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/abffb6d0db3d4ddaa0cdc3bd5e435ca5 is 50, key is test_row_0/C:col10/1731523106394/Put/seqid=0 2024-11-13T18:38:28,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742289_1465 (size=12207) 2024-11-13T18:38:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523168763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523168763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:28,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/58eff25a72af4129956142a7920f302c 2024-11-13T18:38:28,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/860551d749af46c890c3b034898fafad is 50, key is test_row_0/C:col10/1731523107174/Put/seqid=0 2024-11-13T18:38:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742290_1466 (size=12001) 2024-11-13T18:38:28,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:28,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:28,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:28,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:29,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523169084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523169084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:29,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:29,163 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/abffb6d0db3d4ddaa0cdc3bd5e435ca5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/abffb6d0db3d4ddaa0cdc3bd5e435ca5 2024-11-13T18:38:29,167 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into abffb6d0db3d4ddaa0cdc3bd5e435ca5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:29,167 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:29,167 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, priority=13, startTime=1731523108309; duration=0sec 2024-11-13T18:38:29,167 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:29,167 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C 2024-11-13T18:38:29,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/860551d749af46c890c3b034898fafad 2024-11-13T18:38:29,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/cf220dcbe24a45a9b11a0736ef0acb72 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cf220dcbe24a45a9b11a0736ef0acb72 2024-11-13T18:38:29,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cf220dcbe24a45a9b11a0736ef0acb72, entries=200, sequenceid=114, filesize=14.0 K 2024-11-13T18:38:29,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/58eff25a72af4129956142a7920f302c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/58eff25a72af4129956142a7920f302c 2024-11-13T18:38:29,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/58eff25a72af4129956142a7920f302c, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T18:38:29,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/860551d749af46c890c3b034898fafad as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/860551d749af46c890c3b034898fafad 2024-11-13T18:38:29,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/860551d749af46c890c3b034898fafad, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T18:38:29,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 855ms, sequenceid=114, compaction requested=false 2024-11-13T18:38:29,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:29,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T18:38:29,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:29,296 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T18:38:29,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/9f5c4895ee0c41c091e4a072ead101d0 is 50, key is test_row_0/A:col10/1731523108430/Put/seqid=0 2024-11-13T18:38:29,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742291_1467 (size=12001) 2024-11-13T18:38:29,305 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/9f5c4895ee0c41c091e4a072ead101d0 2024-11-13T18:38:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/2ac3c0506a984716baa6c775f83947a1 is 50, key is test_row_0/B:col10/1731523108430/Put/seqid=0 2024-11-13T18:38:29,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742292_1468 (size=12001) 2024-11-13T18:38:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:38:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:29,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:29,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523169550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523169554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523169566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523169603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523169604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523169673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523169683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523169683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,717 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/2ac3c0506a984716baa6c775f83947a1 2024-11-13T18:38:29,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/e5ece69bde1e49d88431f101518e881c is 50, key is test_row_0/C:col10/1731523108430/Put/seqid=0 2024-11-13T18:38:29,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742293_1469 (size=12001) 2024-11-13T18:38:29,727 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/e5ece69bde1e49d88431f101518e881c 2024-11-13T18:38:29,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/9f5c4895ee0c41c091e4a072ead101d0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/9f5c4895ee0c41c091e4a072ead101d0 2024-11-13T18:38:29,735 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/9f5c4895ee0c41c091e4a072ead101d0, entries=150, sequenceid=131, filesize=11.7 K 2024-11-13T18:38:29,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/2ac3c0506a984716baa6c775f83947a1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2ac3c0506a984716baa6c775f83947a1 2024-11-13T18:38:29,738 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2ac3c0506a984716baa6c775f83947a1, entries=150, sequenceid=131, filesize=11.7 K 2024-11-13T18:38:29,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/e5ece69bde1e49d88431f101518e881c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5ece69bde1e49d88431f101518e881c 2024-11-13T18:38:29,742 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5ece69bde1e49d88431f101518e881c, entries=150, sequenceid=131, filesize=11.7 K 2024-11-13T18:38:29,743 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 447ms, sequenceid=131, compaction requested=true 2024-11-13T18:38:29,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:29,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:29,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-13T18:38:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-13T18:38:29,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T18:38:29,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4380 sec 2024-11-13T18:38:29,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.4420 sec 2024-11-13T18:38:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:29,899 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:29,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T18:38:29,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/bb3607d8d8a44b788600c2e0113f1c5b is 50, key is test_row_0/A:col10/1731523109897/Put/seqid=0 2024-11-13T18:38:29,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742294_1470 (size=14541) 2024-11-13T18:38:29,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523169927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523169929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:29,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523169941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523170054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523170054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523170063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523170273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523170273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523170283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/bb3607d8d8a44b788600c2e0113f1c5b 2024-11-13T18:38:30,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7472528352c249bcaa32a9e1e406197e is 50, key is test_row_0/B:col10/1731523109897/Put/seqid=0 2024-11-13T18:38:30,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742295_1471 (size=12151) 2024-11-13T18:38:30,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523170593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523170593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523170603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523170623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523170624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:30,627 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,628 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:30,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7472528352c249bcaa32a9e1e406197e 2024-11-13T18:38:30,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a7f30228d01e4ff1a4eae9b468c242d1 is 50, key is test_row_0/C:col10/1731523109897/Put/seqid=0 2024-11-13T18:38:30,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742296_1472 (size=12151) 2024-11-13T18:38:30,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a7f30228d01e4ff1a4eae9b468c242d1 2024-11-13T18:38:30,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/bb3607d8d8a44b788600c2e0113f1c5b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/bb3607d8d8a44b788600c2e0113f1c5b 2024-11-13T18:38:30,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/bb3607d8d8a44b788600c2e0113f1c5b, entries=200, sequenceid=154, filesize=14.2 K 2024-11-13T18:38:30,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7472528352c249bcaa32a9e1e406197e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7472528352c249bcaa32a9e1e406197e 2024-11-13T18:38:30,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7472528352c249bcaa32a9e1e406197e, entries=150, sequenceid=154, filesize=11.9 K 2024-11-13T18:38:30,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a7f30228d01e4ff1a4eae9b468c242d1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a7f30228d01e4ff1a4eae9b468c242d1 2024-11-13T18:38:30,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a7f30228d01e4ff1a4eae9b468c242d1, entries=150, sequenceid=154, filesize=11.9 K 2024-11-13T18:38:30,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 852ms, sequenceid=154, compaction requested=true 2024-11-13T18:38:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:30,751 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:30,751 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:30,752 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:30,752 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:30,752 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A is initiating minor compaction (all files) 2024-11-13T18:38:30,752 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B is initiating minor compaction (all files) 2024-11-13T18:38:30,752 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:30,752 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:30,752 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7c8b193c26ff4e669ef33d632ba39b13, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cf220dcbe24a45a9b11a0736ef0acb72, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/9f5c4895ee0c41c091e4a072ead101d0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/bb3607d8d8a44b788600c2e0113f1c5b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=51.8 K 2024-11-13T18:38:30,752 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a8da9a759a834755a92c2a5659a78913, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/58eff25a72af4129956142a7920f302c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2ac3c0506a984716baa6c775f83947a1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7472528352c249bcaa32a9e1e406197e] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=47.2 K 2024-11-13T18:38:30,753 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c8b193c26ff4e669ef33d632ba39b13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523106394 2024-11-13T18:38:30,753 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a8da9a759a834755a92c2a5659a78913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523106394 2024-11-13T18:38:30,753 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf220dcbe24a45a9b11a0736ef0acb72, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731523107142 2024-11-13T18:38:30,753 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 58eff25a72af4129956142a7920f302c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731523107174 2024-11-13T18:38:30,753 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f5c4895ee0c41c091e4a072ead101d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731523108397 2024-11-13T18:38:30,753 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac3c0506a984716baa6c775f83947a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731523108397 2024-11-13T18:38:30,754 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb3607d8d8a44b788600c2e0113f1c5b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731523109546 2024-11-13T18:38:30,754 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7472528352c249bcaa32a9e1e406197e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731523109552 2024-11-13T18:38:30,764 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:30,764 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#A#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:30,764 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/a516b3d988764cb2bcb863456c1e39cf is 50, key is test_row_0/B:col10/1731523109897/Put/seqid=0 2024-11-13T18:38:30,764 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/6b74219b814b4b7996f3c56c4c9d1bdb is 50, key is test_row_0/A:col10/1731523109897/Put/seqid=0 2024-11-13T18:38:30,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742297_1473 (size=12493) 2024-11-13T18:38:30,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742298_1474 (size=12493) 2024-11-13T18:38:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:31,121 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:31,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T18:38:31,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/c1b4ab52c8b64326a80f44d1fe7336e7 is 50, key is test_row_0/A:col10/1731523111121/Put/seqid=0 2024-11-13T18:38:31,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742299_1475 (size=16931) 2024-11-13T18:38:31,175 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/a516b3d988764cb2bcb863456c1e39cf as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a516b3d988764cb2bcb863456c1e39cf 2024-11-13T18:38:31,182 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into a516b3d988764cb2bcb863456c1e39cf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:31,182 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:31,182 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, priority=12, startTime=1731523110751; duration=0sec 2024-11-13T18:38:31,182 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:31,182 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B 2024-11-13T18:38:31,182 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:31,184 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:31,184 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C is initiating minor compaction (all files) 2024-11-13T18:38:31,184 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:31,184 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/abffb6d0db3d4ddaa0cdc3bd5e435ca5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/860551d749af46c890c3b034898fafad, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5ece69bde1e49d88431f101518e881c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a7f30228d01e4ff1a4eae9b468c242d1] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=47.2 K 2024-11-13T18:38:31,185 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting abffb6d0db3d4ddaa0cdc3bd5e435ca5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731523106394 2024-11-13T18:38:31,185 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 860551d749af46c890c3b034898fafad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731523107174 2024-11-13T18:38:31,185 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/6b74219b814b4b7996f3c56c4c9d1bdb as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6b74219b814b4b7996f3c56c4c9d1bdb 2024-11-13T18:38:31,186 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ece69bde1e49d88431f101518e881c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731523108397 2024-11-13T18:38:31,186 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f30228d01e4ff1a4eae9b468c242d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731523109552 2024-11-13T18:38:31,189 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 6b74219b814b4b7996f3c56c4c9d1bdb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:31,189 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:31,189 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, priority=12, startTime=1731523110751; duration=0sec 2024-11-13T18:38:31,189 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:31,189 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A 2024-11-13T18:38:31,193 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:31,194 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/32567ad08e6c4b64a0d641d016348842 is 50, key is test_row_0/C:col10/1731523109897/Put/seqid=0 2024-11-13T18:38:31,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742300_1476 (size=12493) 2024-11-13T18:38:31,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523171198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523171199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523171203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523171314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523171323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523171323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T18:38:31,463 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:31,464 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-13T18:38:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:38:31,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:31,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:31,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:31,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523171524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/c1b4ab52c8b64326a80f44d1fe7336e7 2024-11-13T18:38:31,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/9d78004d7a484593921adafd51215191 is 50, key is test_row_0/B:col10/1731523111121/Put/seqid=0 2024-11-13T18:38:31,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742301_1477 (size=12151) 2024-11-13T18:38:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523171543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523171543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:38:31,601 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/32567ad08e6c4b64a0d641d016348842 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/32567ad08e6c4b64a0d641d016348842 2024-11-13T18:38:31,605 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 32567ad08e6c4b64a0d641d016348842(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:31,605 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:31,605 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, priority=12, startTime=1731523110751; duration=0sec 2024-11-13T18:38:31,605 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:31,605 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C 2024-11-13T18:38:31,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:38:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:31,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:38:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:31,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:38:31,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523171843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523171863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523171864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:31,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:38:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:31,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:31,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/9d78004d7a484593921adafd51215191 2024-11-13T18:38:31,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ef647031420144138f584e83e43a3045 is 50, key is test_row_0/C:col10/1731523111121/Put/seqid=0 2024-11-13T18:38:31,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742302_1478 (size=12151) 2024-11-13T18:38:31,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ef647031420144138f584e83e43a3045 2024-11-13T18:38:31,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/c1b4ab52c8b64326a80f44d1fe7336e7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/c1b4ab52c8b64326a80f44d1fe7336e7 2024-11-13T18:38:31,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/c1b4ab52c8b64326a80f44d1fe7336e7, entries=250, sequenceid=169, filesize=16.5 K 2024-11-13T18:38:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/9d78004d7a484593921adafd51215191 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9d78004d7a484593921adafd51215191 2024-11-13T18:38:31,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9d78004d7a484593921adafd51215191, entries=150, sequenceid=169, filesize=11.9 K 2024-11-13T18:38:31,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ef647031420144138f584e83e43a3045 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ef647031420144138f584e83e43a3045 2024-11-13T18:38:31,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ef647031420144138f584e83e43a3045, entries=150, sequenceid=169, filesize=11.9 K 2024-11-13T18:38:31,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 857ms, sequenceid=169, compaction requested=false 2024-11-13T18:38:31,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:32,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T18:38:32,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:32,078 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T18:38:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/19353a30bdce43c1834a10d01b5740fc is 50, key is test_row_0/A:col10/1731523111198/Put/seqid=0 2024-11-13T18:38:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:38:32,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742303_1479 (size=12151) 2024-11-13T18:38:32,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:32,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523172414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523172415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523172426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,495 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/19353a30bdce43c1834a10d01b5740fc 2024-11-13T18:38:32,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7c2ab8c934104245b1700b69ecbb26de is 50, key is test_row_0/B:col10/1731523111198/Put/seqid=0 2024-11-13T18:38:32,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742304_1480 (size=12151) 2024-11-13T18:38:32,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523172533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523172533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523172533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:38:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523172643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523172643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,655 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,656 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523172743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523172754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523172754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:32,906 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7c2ab8c934104245b1700b69ecbb26de 2024-11-13T18:38:32,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/4ff274c1aa3f470ea7492bbe04d6d189 is 50, key is test_row_0/C:col10/1731523111198/Put/seqid=0 2024-11-13T18:38:32,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742305_1481 (size=12151) 2024-11-13T18:38:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523173053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523173073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523173073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,317 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/4ff274c1aa3f470ea7492bbe04d6d189 2024-11-13T18:38:33,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/19353a30bdce43c1834a10d01b5740fc as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/19353a30bdce43c1834a10d01b5740fc 2024-11-13T18:38:33,332 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/19353a30bdce43c1834a10d01b5740fc, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T18:38:33,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7c2ab8c934104245b1700b69ecbb26de as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7c2ab8c934104245b1700b69ecbb26de 2024-11-13T18:38:33,337 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7c2ab8c934104245b1700b69ecbb26de, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T18:38:33,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/4ff274c1aa3f470ea7492bbe04d6d189 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/4ff274c1aa3f470ea7492bbe04d6d189 2024-11-13T18:38:33,341 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/4ff274c1aa3f470ea7492bbe04d6d189, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T18:38:33,344 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 1267ms, sequenceid=193, compaction requested=true 2024-11-13T18:38:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-13T18:38:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-13T18:38:33,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T18:38:33,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8800 sec 2024-11-13T18:38:33,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8840 sec 2024-11-13T18:38:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:33,581 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:33,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T18:38:33,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/3d60a8fe060644fd806351b05be40043 is 50, key is test_row_0/A:col10/1731523112424/Put/seqid=0 2024-11-13T18:38:33,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742306_1482 (size=14541) 2024-11-13T18:38:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T18:38:33,614 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:33,614 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-13T18:38:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:38:33,616 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:33,616 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:33,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:33,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523173655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523173656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523173663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:38:33,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:33,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:33,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523173773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523173783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523173783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:33,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:33,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:33,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:33,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:33,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:33,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:38:33,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/3d60a8fe060644fd806351b05be40043 2024-11-13T18:38:33,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/17e25aafaf72417497cd55a229d45509 is 50, key is test_row_0/B:col10/1731523112424/Put/seqid=0 2024-11-13T18:38:34,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742307_1483 (size=12151) 2024-11-13T18:38:34,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523173993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523173993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523173993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:34,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:34,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:34,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:34,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:38:34,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523174303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523174314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523174314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:34,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:34,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/17e25aafaf72417497cd55a229d45509 2024-11-13T18:38:34,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/677dc7e021074d009ed4969b5669d817 is 50, key is test_row_0/C:col10/1731523112424/Put/seqid=0 2024-11-13T18:38:34,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742308_1484 (size=12151) 2024-11-13T18:38:34,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:34,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:34,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:38:34,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/677dc7e021074d009ed4969b5669d817 2024-11-13T18:38:34,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/3d60a8fe060644fd806351b05be40043 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/3d60a8fe060644fd806351b05be40043 2024-11-13T18:38:34,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/3d60a8fe060644fd806351b05be40043, entries=200, sequenceid=209, filesize=14.2 K 2024-11-13T18:38:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/17e25aafaf72417497cd55a229d45509 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/17e25aafaf72417497cd55a229d45509 2024-11-13T18:38:34,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/17e25aafaf72417497cd55a229d45509, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T18:38:34,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/677dc7e021074d009ed4969b5669d817 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/677dc7e021074d009ed4969b5669d817 2024-11-13T18:38:34,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/677dc7e021074d009ed4969b5669d817, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T18:38:34,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 1247ms, sequenceid=209, compaction requested=true 2024-11-13T18:38:34,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:34,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:34,828 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:34,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:34,828 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:34,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:34,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:34,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:34,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:34,829 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:34,829 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:34,829 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B is initiating minor compaction (all files) 2024-11-13T18:38:34,829 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A is initiating minor compaction (all files) 2024-11-13T18:38:34,829 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,829 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,829 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a516b3d988764cb2bcb863456c1e39cf, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9d78004d7a484593921adafd51215191, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7c2ab8c934104245b1700b69ecbb26de, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/17e25aafaf72417497cd55a229d45509] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=47.8 K 2024-11-13T18:38:34,829 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6b74219b814b4b7996f3c56c4c9d1bdb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/c1b4ab52c8b64326a80f44d1fe7336e7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/19353a30bdce43c1834a10d01b5740fc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/3d60a8fe060644fd806351b05be40043] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=54.8 K 2024-11-13T18:38:34,829 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a516b3d988764cb2bcb863456c1e39cf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731523109552 2024-11-13T18:38:34,830 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b74219b814b4b7996f3c56c4c9d1bdb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731523109552 2024-11-13T18:38:34,830 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d78004d7a484593921adafd51215191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523109925 2024-11-13T18:38:34,830 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b4ab52c8b64326a80f44d1fe7336e7, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523109923 2024-11-13T18:38:34,830 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19353a30bdce43c1834a10d01b5740fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731523111185 2024-11-13T18:38:34,830 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c2ab8c934104245b1700b69ecbb26de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731523111185 2024-11-13T18:38:34,830 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d60a8fe060644fd806351b05be40043, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731523112408 2024-11-13T18:38:34,830 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 17e25aafaf72417497cd55a229d45509, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731523112414 2024-11-13T18:38:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:34,838 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:34,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T18:38:34,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,844 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#B#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:34,844 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/b3fb99bd4cf64b4393972ed6eeb8c2ca is 50, key is test_row_0/B:col10/1731523112424/Put/seqid=0 2024-11-13T18:38:34,846 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#A#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:34,847 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/df82440f6f614788ae68272445f937a3 is 50, key is test_row_0/A:col10/1731523112424/Put/seqid=0 2024-11-13T18:38:34,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/7ccc7ede5d0e46f1b4cab3df14e452c4 is 50, key is test_row_0/A:col10/1731523113653/Put/seqid=0 2024-11-13T18:38:34,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742310_1486 (size=12629) 2024-11-13T18:38:34,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742309_1485 (size=12629) 2024-11-13T18:38:34,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742311_1487 (size=14541) 2024-11-13T18:38:34,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/7ccc7ede5d0e46f1b4cab3df14e452c4 2024-11-13T18:38:34,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/ade82f0cce9d450e8e32634fb60c7152 is 50, key is test_row_0/B:col10/1731523113653/Put/seqid=0 2024-11-13T18:38:34,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742312_1488 (size=12151) 2024-11-13T18:38:34,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523174893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523174895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523174895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:34,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:34,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:34,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:34,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523175003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523175023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523175023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:35,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:35,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523175223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523175244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523175244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,265 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/df82440f6f614788ae68272445f937a3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/df82440f6f614788ae68272445f937a3 2024-11-13T18:38:35,265 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/b3fb99bd4cf64b4393972ed6eeb8c2ca as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/b3fb99bd4cf64b4393972ed6eeb8c2ca 2024-11-13T18:38:35,274 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into b3fb99bd4cf64b4393972ed6eeb8c2ca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:35,274 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:35,274 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, priority=12, startTime=1731523114828; duration=0sec 2024-11-13T18:38:35,274 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:35,274 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B 2024-11-13T18:38:35,275 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:35,275 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into df82440f6f614788ae68272445f937a3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:35,275 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:35,275 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, priority=12, startTime=1731523114828; duration=0sec 2024-11-13T18:38:35,275 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:35,275 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A 2024-11-13T18:38:35,277 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:35,277 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C is initiating minor compaction (all files) 2024-11-13T18:38:35,277 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,277 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/32567ad08e6c4b64a0d641d016348842, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ef647031420144138f584e83e43a3045, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/4ff274c1aa3f470ea7492bbe04d6d189, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/677dc7e021074d009ed4969b5669d817] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=47.8 K 2024-11-13T18:38:35,278 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 32567ad08e6c4b64a0d641d016348842, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731523109552 2024-11-13T18:38:35,278 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ef647031420144138f584e83e43a3045, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731523109925 2024-11-13T18:38:35,278 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ff274c1aa3f470ea7492bbe04d6d189, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731523111185 2024-11-13T18:38:35,279 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 677dc7e021074d009ed4969b5669d817, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731523112414 2024-11-13T18:38:35,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/ade82f0cce9d450e8e32634fb60c7152 2024-11-13T18:38:35,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,309 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#C#compaction#414 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:35,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/33da2cf8dc7e49b1b0b662f4196c64ff is 50, key is test_row_0/C:col10/1731523113653/Put/seqid=0 2024-11-13T18:38:35,312 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/e5c4b6bc8a974fa784592cec80150fc2 is 50, key is test_row_0/C:col10/1731523112424/Put/seqid=0 2024-11-13T18:38:35,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742313_1489 (size=12151) 2024-11-13T18:38:35,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742314_1490 (size=12629) 2024-11-13T18:38:35,359 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/e5c4b6bc8a974fa784592cec80150fc2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5c4b6bc8a974fa784592cec80150fc2 2024-11-13T18:38:35,363 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into e5c4b6bc8a974fa784592cec80150fc2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:35,363 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:35,364 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, priority=12, startTime=1731523114828; duration=0sec 2024-11-13T18:38:35,364 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:35,364 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C 2024-11-13T18:38:35,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:35,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523175543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523175573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523175574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:35,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:35,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:35,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/33da2cf8dc7e49b1b0b662f4196c64ff 2024-11-13T18:38:35,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:35,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:35,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:35,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/7ccc7ede5d0e46f1b4cab3df14e452c4 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7ccc7ede5d0e46f1b4cab3df14e452c4 2024-11-13T18:38:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:38:35,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7ccc7ede5d0e46f1b4cab3df14e452c4, entries=200, sequenceid=230, filesize=14.2 K 2024-11-13T18:38:35,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/ade82f0cce9d450e8e32634fb60c7152 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ade82f0cce9d450e8e32634fb60c7152 2024-11-13T18:38:35,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ade82f0cce9d450e8e32634fb60c7152, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T18:38:35,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/33da2cf8dc7e49b1b0b662f4196c64ff as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/33da2cf8dc7e49b1b0b662f4196c64ff 2024-11-13T18:38:35,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/33da2cf8dc7e49b1b0b662f4196c64ff, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T18:38:35,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 940ms, sequenceid=230, compaction requested=false 2024-11-13T18:38:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:35,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T18:38:35,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:35,913 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T18:38:35,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ea0a07a5fbe24582999e9b6078a0c242 is 50, key is test_row_0/A:col10/1731523114874/Put/seqid=0 2024-11-13T18:38:35,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742315_1491 (size=12151) 2024-11-13T18:38:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:36,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:36,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523176153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523176163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523176164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523176273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523176273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523176283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,345 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ea0a07a5fbe24582999e9b6078a0c242 2024-11-13T18:38:36,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/340e98c20ab34d53a73e574922859947 is 50, key is test_row_0/B:col10/1731523114874/Put/seqid=0 2024-11-13T18:38:36,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742316_1492 (size=12151) 2024-11-13T18:38:36,376 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/340e98c20ab34d53a73e574922859947 2024-11-13T18:38:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ebe827a01c3149c6b7d365386ba5cb5e is 50, key is test_row_0/C:col10/1731523114874/Put/seqid=0 2024-11-13T18:38:36,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742317_1493 (size=12151) 2024-11-13T18:38:36,390 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ebe827a01c3149c6b7d365386ba5cb5e 2024-11-13T18:38:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ea0a07a5fbe24582999e9b6078a0c242 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea0a07a5fbe24582999e9b6078a0c242 2024-11-13T18:38:36,397 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea0a07a5fbe24582999e9b6078a0c242, entries=150, sequenceid=249, filesize=11.9 K 2024-11-13T18:38:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/340e98c20ab34d53a73e574922859947 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/340e98c20ab34d53a73e574922859947 2024-11-13T18:38:36,402 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/340e98c20ab34d53a73e574922859947, entries=150, sequenceid=249, filesize=11.9 K 2024-11-13T18:38:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ebe827a01c3149c6b7d365386ba5cb5e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ebe827a01c3149c6b7d365386ba5cb5e 2024-11-13T18:38:36,409 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ebe827a01c3149c6b7d365386ba5cb5e, entries=150, sequenceid=249, filesize=11.9 K 2024-11-13T18:38:36,410 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 497ms, sequenceid=249, compaction requested=true 2024-11-13T18:38:36,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:36,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:36,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-13T18:38:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-13T18:38:36,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T18:38:36,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7960 sec 2024-11-13T18:38:36,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.8000 sec 2024-11-13T18:38:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:36,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:36,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T18:38:36,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/47e4a52f694449ad99cea3c299c88640 is 50, key is test_row_0/A:col10/1731523116488/Put/seqid=0 2024-11-13T18:38:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742318_1494 (size=14741) 2024-11-13T18:38:36,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/47e4a52f694449ad99cea3c299c88640 2024-11-13T18:38:36,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/63efc8fc342c43f8bbae5531c8426ade is 50, key is test_row_0/B:col10/1731523116488/Put/seqid=0 2024-11-13T18:38:36,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742319_1495 (size=12301) 2024-11-13T18:38:36,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523176540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523176541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523176543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523176654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523176654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523176654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43682 deadline: 1731523176673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,684 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43612 deadline: 1731523176693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,708 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523176873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523176873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523176873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:36,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/63efc8fc342c43f8bbae5531c8426ade 2024-11-13T18:38:36,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/0abccc05161747eabec1277cd166f221 is 50, key is test_row_0/C:col10/1731523116488/Put/seqid=0 2024-11-13T18:38:36,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742320_1496 (size=12301) 2024-11-13T18:38:36,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/0abccc05161747eabec1277cd166f221 2024-11-13T18:38:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/47e4a52f694449ad99cea3c299c88640 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47e4a52f694449ad99cea3c299c88640 2024-11-13T18:38:36,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47e4a52f694449ad99cea3c299c88640, entries=200, sequenceid=271, filesize=14.4 K 2024-11-13T18:38:36,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/63efc8fc342c43f8bbae5531c8426ade as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/63efc8fc342c43f8bbae5531c8426ade 2024-11-13T18:38:36,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/63efc8fc342c43f8bbae5531c8426ade, entries=150, sequenceid=271, filesize=12.0 K 2024-11-13T18:38:36,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/0abccc05161747eabec1277cd166f221 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0abccc05161747eabec1277cd166f221 2024-11-13T18:38:36,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0abccc05161747eabec1277cd166f221, entries=150, sequenceid=271, filesize=12.0 K 2024-11-13T18:38:36,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 475ms, sequenceid=271, compaction requested=true 2024-11-13T18:38:36,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:36,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:36,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:36,966 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:36,966 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:36,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:36,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:36,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:36,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:36,967 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:36,967 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B is initiating minor compaction (all files) 2024-11-13T18:38:36,967 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:36,967 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/b3fb99bd4cf64b4393972ed6eeb8c2ca, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ade82f0cce9d450e8e32634fb60c7152, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/340e98c20ab34d53a73e574922859947, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/63efc8fc342c43f8bbae5531c8426ade] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=48.1 K 2024-11-13T18:38:36,967 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b3fb99bd4cf64b4393972ed6eeb8c2ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731523112414 2024-11-13T18:38:36,968 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ade82f0cce9d450e8e32634fb60c7152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731523113653 2024-11-13T18:38:36,968 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 340e98c20ab34d53a73e574922859947, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731523114873 2024-11-13T18:38:36,968 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 63efc8fc342c43f8bbae5531c8426ade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731523116146 2024-11-13T18:38:36,968 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:36,968 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A is initiating minor compaction (all files) 2024-11-13T18:38:36,968 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:36,969 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/df82440f6f614788ae68272445f937a3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7ccc7ede5d0e46f1b4cab3df14e452c4, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea0a07a5fbe24582999e9b6078a0c242, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47e4a52f694449ad99cea3c299c88640] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=52.8 K 2024-11-13T18:38:36,969 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting df82440f6f614788ae68272445f937a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731523112414 2024-11-13T18:38:36,969 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ccc7ede5d0e46f1b4cab3df14e452c4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731523113653 2024-11-13T18:38:36,970 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea0a07a5fbe24582999e9b6078a0c242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731523114873 2024-11-13T18:38:36,970 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47e4a52f694449ad99cea3c299c88640, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731523116146 2024-11-13T18:38:36,999 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#B#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:37,000 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7035eec604f94cc090630b284481545b is 50, key is test_row_0/B:col10/1731523116488/Put/seqid=0 2024-11-13T18:38:37,000 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:37,000 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/4a380823cb8b4f43b124682e6b9a9bed is 50, key is test_row_0/A:col10/1731523116488/Put/seqid=0 2024-11-13T18:38:37,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742321_1497 (size=12915) 2024-11-13T18:38:37,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742322_1498 (size=12915) 2024-11-13T18:38:37,018 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/4a380823cb8b4f43b124682e6b9a9bed as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/4a380823cb8b4f43b124682e6b9a9bed 2024-11-13T18:38:37,023 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 4a380823cb8b4f43b124682e6b9a9bed(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:37,023 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:37,023 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, priority=12, startTime=1731523116966; duration=0sec 2024-11-13T18:38:37,023 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:37,023 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A 2024-11-13T18:38:37,023 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:38:37,024 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T18:38:37,024 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C is initiating minor compaction (all files) 2024-11-13T18:38:37,024 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:37,024 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5c4b6bc8a974fa784592cec80150fc2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/33da2cf8dc7e49b1b0b662f4196c64ff, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ebe827a01c3149c6b7d365386ba5cb5e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0abccc05161747eabec1277cd166f221] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=48.1 K 2024-11-13T18:38:37,024 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5c4b6bc8a974fa784592cec80150fc2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731523112414 2024-11-13T18:38:37,025 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33da2cf8dc7e49b1b0b662f4196c64ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731523113653 2024-11-13T18:38:37,025 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe827a01c3149c6b7d365386ba5cb5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731523114873 2024-11-13T18:38:37,025 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0abccc05161747eabec1277cd166f221, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731523116146 2024-11-13T18:38:37,035 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:37,036 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ba6b2f0df19f48d3a964590a6c1e9b9f is 50, key is test_row_0/C:col10/1731523116488/Put/seqid=0 2024-11-13T18:38:37,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742323_1499 (size=12915) 2024-11-13T18:38:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:37,201 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:37,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T18:38:37,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/996f325569ec434dad1ee13ff8d99b97 is 50, key is test_row_0/A:col10/1731523116541/Put/seqid=0 2024-11-13T18:38:37,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742324_1500 (size=17181) 2024-11-13T18:38:37,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523177272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523177274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523177283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523177393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523177393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523177403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,418 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7035eec604f94cc090630b284481545b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7035eec604f94cc090630b284481545b 2024-11-13T18:38:37,423 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 7035eec604f94cc090630b284481545b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:37,423 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:37,423 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, priority=12, startTime=1731523116966; duration=0sec 2024-11-13T18:38:37,423 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:37,423 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B 2024-11-13T18:38:37,449 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ba6b2f0df19f48d3a964590a6c1e9b9f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ba6b2f0df19f48d3a964590a6c1e9b9f 2024-11-13T18:38:37,456 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into ba6b2f0df19f48d3a964590a6c1e9b9f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:37,456 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:37,456 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, priority=12, startTime=1731523116966; duration=0sec 2024-11-13T18:38:37,456 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:37,456 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C 2024-11-13T18:38:37,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/996f325569ec434dad1ee13ff8d99b97 2024-11-13T18:38:37,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/0ea4e5473e3f49cebca8b9771fb0bfba is 50, key is test_row_0/B:col10/1731523116541/Put/seqid=0 2024-11-13T18:38:37,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523177613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742325_1501 (size=12301) 2024-11-13T18:38:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523177613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/0ea4e5473e3f49cebca8b9771fb0bfba 2024-11-13T18:38:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523177623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1cbd39c3f54b46698ea7be644dd71402 is 50, key is test_row_0/C:col10/1731523116541/Put/seqid=0 2024-11-13T18:38:37,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742326_1502 (size=12301) 2024-11-13T18:38:37,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1cbd39c3f54b46698ea7be644dd71402 2024-11-13T18:38:37,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/996f325569ec434dad1ee13ff8d99b97 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/996f325569ec434dad1ee13ff8d99b97 2024-11-13T18:38:37,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/996f325569ec434dad1ee13ff8d99b97, entries=250, sequenceid=288, filesize=16.8 K 2024-11-13T18:38:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/0ea4e5473e3f49cebca8b9771fb0bfba as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/0ea4e5473e3f49cebca8b9771fb0bfba 2024-11-13T18:38:37,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/0ea4e5473e3f49cebca8b9771fb0bfba, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T18:38:37,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1cbd39c3f54b46698ea7be644dd71402 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1cbd39c3f54b46698ea7be644dd71402 2024-11-13T18:38:37,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1cbd39c3f54b46698ea7be644dd71402, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T18:38:37,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 465ms, sequenceid=288, compaction requested=false 2024-11-13T18:38:37,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T18:38:37,773 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:37,774 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-13T18:38:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:38:37,778 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:37,779 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:37,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:38:37,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T18:38:37,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:37,931 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T18:38:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:37,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/a2861409b1fb4bf58a930253e1a05994 is 50, key is test_row_0/A:col10/1731523117273/Put/seqid=0 2024-11-13T18:38:37,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742327_1503 (size=12301) 2024-11-13T18:38:37,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523177974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523177987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523177989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:38:38,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523178093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523178104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523178113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523178314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523178323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523178323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,349 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/a2861409b1fb4bf58a930253e1a05994 2024-11-13T18:38:38,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/8cd389a16e244b2eb74a3fbbe1cb3269 is 50, key is test_row_0/B:col10/1731523117273/Put/seqid=0 2024-11-13T18:38:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742328_1504 (size=12301) 2024-11-13T18:38:38,368 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/8cd389a16e244b2eb74a3fbbe1cb3269 2024-11-13T18:38:38,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/bc765f6e92144629b2f4d54fdac0f56d is 50, key is test_row_0/C:col10/1731523117273/Put/seqid=0 2024-11-13T18:38:38,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742329_1505 (size=12301) 2024-11-13T18:38:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:38:38,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523178634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523178634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523178643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:38,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:38,791 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/bc765f6e92144629b2f4d54fdac0f56d 2024-11-13T18:38:38,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/a2861409b1fb4bf58a930253e1a05994 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a2861409b1fb4bf58a930253e1a05994 2024-11-13T18:38:38,799 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a2861409b1fb4bf58a930253e1a05994, entries=150, sequenceid=310, filesize=12.0 K 2024-11-13T18:38:38,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/8cd389a16e244b2eb74a3fbbe1cb3269 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8cd389a16e244b2eb74a3fbbe1cb3269 2024-11-13T18:38:38,805 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8cd389a16e244b2eb74a3fbbe1cb3269, entries=150, sequenceid=310, filesize=12.0 K 2024-11-13T18:38:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/bc765f6e92144629b2f4d54fdac0f56d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/bc765f6e92144629b2f4d54fdac0f56d 2024-11-13T18:38:38,809 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/bc765f6e92144629b2f4d54fdac0f56d, entries=150, sequenceid=310, filesize=12.0 K 2024-11-13T18:38:38,810 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 879ms, sequenceid=310, compaction requested=true 2024-11-13T18:38:38,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:38,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:38,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-13T18:38:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-13T18:38:38,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T18:38:38,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0370 sec 2024-11-13T18:38:38,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0440 sec 2024-11-13T18:38:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T18:38:38,914 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:38,914 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-13T18:38:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:38:38,916 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:38,917 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:38,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:38:39,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T18:38:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:39,069 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T18:38:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/24828cdcfe574be7becc76c1cff37892 is 50, key is test_row_0/A:col10/1731523117966/Put/seqid=0 2024-11-13T18:38:39,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742330_1506 (size=12301) 2024-11-13T18:38:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:39,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:38:39,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523179219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523179223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523179237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523179344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523179344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523179363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,481 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/24828cdcfe574be7becc76c1cff37892 2024-11-13T18:38:39,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/2bf9d72df863401ba8442fd4e73ba0bd is 50, key is test_row_0/B:col10/1731523117966/Put/seqid=0 2024-11-13T18:38:39,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742331_1507 (size=12301) 2024-11-13T18:38:39,501 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/2bf9d72df863401ba8442fd4e73ba0bd 2024-11-13T18:38:39,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/2a8545889d7143daaa02e713ed7092f8 is 50, key is test_row_0/C:col10/1731523117966/Put/seqid=0 2024-11-13T18:38:39,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742332_1508 (size=12301) 2024-11-13T18:38:39,519 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/2a8545889d7143daaa02e713ed7092f8 2024-11-13T18:38:39,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/24828cdcfe574be7becc76c1cff37892 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/24828cdcfe574be7becc76c1cff37892 2024-11-13T18:38:39,532 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/24828cdcfe574be7becc76c1cff37892, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T18:38:39,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/2bf9d72df863401ba8442fd4e73ba0bd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2bf9d72df863401ba8442fd4e73ba0bd 2024-11-13T18:38:39,540 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2bf9d72df863401ba8442fd4e73ba0bd, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T18:38:39,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/2a8545889d7143daaa02e713ed7092f8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/2a8545889d7143daaa02e713ed7092f8 2024-11-13T18:38:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:38:39,545 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/2a8545889d7143daaa02e713ed7092f8, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T18:38:39,546 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 477ms, sequenceid=326, compaction requested=true 2024-11-13T18:38:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-13T18:38:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-13T18:38:39,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T18:38:39,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 631 msec 2024-11-13T18:38:39,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 636 msec 2024-11-13T18:38:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:39,579 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:39,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T18:38:39,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/5cbdbf56ab074f4a84ff3c1e9ace0b86 is 50, key is test_row_0/A:col10/1731523119221/Put/seqid=0 2024-11-13T18:38:39,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742333_1509 (size=14741) 2024-11-13T18:38:39,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523179642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523179646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523179648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523179764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523179773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523179773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:39,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523179983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523179993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523179993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/5cbdbf56ab074f4a84ff3c1e9ace0b86 2024-11-13T18:38:40,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/200b22ca7dfa4598820167f720beb410 is 50, key is test_row_0/B:col10/1731523119221/Put/seqid=0 2024-11-13T18:38:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742334_1510 (size=12301) 2024-11-13T18:38:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T18:38:40,054 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:40,054 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-13T18:38:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:38:40,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:40,057 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:40,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:38:40,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T18:38:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523180303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523180313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:40,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523180313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T18:38:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:38:40,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/200b22ca7dfa4598820167f720beb410 2024-11-13T18:38:40,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1b71bc16499c442fa616242debacc21f is 50, key is test_row_0/C:col10/1731523119221/Put/seqid=0 2024-11-13T18:38:40,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742335_1511 (size=12301) 2024-11-13T18:38:40,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T18:38:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T18:38:40,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:40,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:38:40,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T18:38:40,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:40,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523180823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523180833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:40,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523180833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:40,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1b71bc16499c442fa616242debacc21f 2024-11-13T18:38:40,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/5cbdbf56ab074f4a84ff3c1e9ace0b86 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/5cbdbf56ab074f4a84ff3c1e9ace0b86 2024-11-13T18:38:40,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/5cbdbf56ab074f4a84ff3c1e9ace0b86, entries=200, sequenceid=347, filesize=14.4 K 2024-11-13T18:38:40,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/200b22ca7dfa4598820167f720beb410 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/200b22ca7dfa4598820167f720beb410 2024-11-13T18:38:40,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/200b22ca7dfa4598820167f720beb410, entries=150, sequenceid=347, filesize=12.0 K 2024-11-13T18:38:40,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1b71bc16499c442fa616242debacc21f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1b71bc16499c442fa616242debacc21f 2024-11-13T18:38:40,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1b71bc16499c442fa616242debacc21f, entries=150, sequenceid=347, filesize=12.0 K 2024-11-13T18:38:40,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 1286ms, sequenceid=347, compaction requested=true 2024-11-13T18:38:40,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:40,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:40,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:40,865 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T18:38:40,865 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T18:38:40,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:40,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:40,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:40,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:40,867 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62119 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T18:38:40,867 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 69439 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T18:38:40,867 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B is initiating minor compaction (all files) 2024-11-13T18:38:40,867 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A is initiating minor compaction (all files) 2024-11-13T18:38:40,867 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,867 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/4a380823cb8b4f43b124682e6b9a9bed, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/996f325569ec434dad1ee13ff8d99b97, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a2861409b1fb4bf58a930253e1a05994, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/24828cdcfe574be7becc76c1cff37892, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/5cbdbf56ab074f4a84ff3c1e9ace0b86] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=67.8 K 2024-11-13T18:38:40,867 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,867 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7035eec604f94cc090630b284481545b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/0ea4e5473e3f49cebca8b9771fb0bfba, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8cd389a16e244b2eb74a3fbbe1cb3269, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2bf9d72df863401ba8442fd4e73ba0bd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/200b22ca7dfa4598820167f720beb410] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=60.7 K 2024-11-13T18:38:40,868 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a380823cb8b4f43b124682e6b9a9bed, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731523116146 2024-11-13T18:38:40,868 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7035eec604f94cc090630b284481545b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731523116146 2024-11-13T18:38:40,868 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 996f325569ec434dad1ee13ff8d99b97, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731523116512 2024-11-13T18:38:40,868 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ea4e5473e3f49cebca8b9771fb0bfba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731523116512 2024-11-13T18:38:40,868 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2861409b1fb4bf58a930253e1a05994, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731523117252 2024-11-13T18:38:40,868 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cd389a16e244b2eb74a3fbbe1cb3269, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731523117252 2024-11-13T18:38:40,869 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24828cdcfe574be7becc76c1cff37892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731523117966 2024-11-13T18:38:40,869 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf9d72df863401ba8442fd4e73ba0bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731523117966 2024-11-13T18:38:40,869 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 200b22ca7dfa4598820167f720beb410, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731523119216 2024-11-13T18:38:40,869 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbdbf56ab074f4a84ff3c1e9ace0b86, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731523119216 2024-11-13T18:38:40,878 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:40,878 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/564062c5e9ad401eac1cad8bd2c33752 is 50, key is test_row_0/A:col10/1731523119221/Put/seqid=0 2024-11-13T18:38:40,884 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:40,884 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/ba55587565904131b99eb757cbec7aff is 50, key is test_row_0/B:col10/1731523119221/Put/seqid=0 2024-11-13T18:38:40,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742337_1513 (size=13085) 2024-11-13T18:38:40,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742336_1512 (size=13085) 2024-11-13T18:38:40,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T18:38:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:40,976 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T18:38:40,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/27869b390ec348f0924b70ea277181b0 is 50, key is test_row_0/A:col10/1731523119646/Put/seqid=0 2024-11-13T18:38:40,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742338_1514 (size=12301) 2024-11-13T18:38:40,986 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/27869b390ec348f0924b70ea277181b0 2024-11-13T18:38:40,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/82157507cd93412e8ceee03a4ca040ff is 50, key is test_row_0/B:col10/1731523119646/Put/seqid=0 2024-11-13T18:38:41,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742339_1515 (size=12301) 2024-11-13T18:38:41,002 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/82157507cd93412e8ceee03a4ca040ff 2024-11-13T18:38:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/b3108ac6a7264413a6816379d97e89c0 is 50, key is test_row_0/C:col10/1731523119646/Put/seqid=0 2024-11-13T18:38:41,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742340_1516 (size=12301) 2024-11-13T18:38:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:38:41,314 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/ba55587565904131b99eb757cbec7aff as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ba55587565904131b99eb757cbec7aff 2024-11-13T18:38:41,318 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into ba55587565904131b99eb757cbec7aff(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:41,318 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:41,318 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, priority=11, startTime=1731523120865; duration=0sec 2024-11-13T18:38:41,318 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:41,318 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B 2024-11-13T18:38:41,318 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T18:38:41,318 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/564062c5e9ad401eac1cad8bd2c33752 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/564062c5e9ad401eac1cad8bd2c33752 2024-11-13T18:38:41,319 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62119 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T18:38:41,319 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C is initiating minor compaction (all files) 2024-11-13T18:38:41,320 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:41,320 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ba6b2f0df19f48d3a964590a6c1e9b9f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1cbd39c3f54b46698ea7be644dd71402, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/bc765f6e92144629b2f4d54fdac0f56d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/2a8545889d7143daaa02e713ed7092f8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1b71bc16499c442fa616242debacc21f] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=60.7 K 2024-11-13T18:38:41,320 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ba6b2f0df19f48d3a964590a6c1e9b9f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731523116146 2024-11-13T18:38:41,320 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cbd39c3f54b46698ea7be644dd71402, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731523116512 2024-11-13T18:38:41,321 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting bc765f6e92144629b2f4d54fdac0f56d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731523117252 2024-11-13T18:38:41,321 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8545889d7143daaa02e713ed7092f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731523117966 2024-11-13T18:38:41,322 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b71bc16499c442fa616242debacc21f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731523119216 2024-11-13T18:38:41,322 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 564062c5e9ad401eac1cad8bd2c33752(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:41,322 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:41,322 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, priority=11, startTime=1731523120865; duration=0sec 2024-11-13T18:38:41,322 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:41,322 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A 2024-11-13T18:38:41,329 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#C#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:41,330 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/adc622c526454128ac0e37c9a01f8385 is 50, key is test_row_0/C:col10/1731523119221/Put/seqid=0 2024-11-13T18:38:41,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742341_1517 (size=13085) 2024-11-13T18:38:41,415 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/b3108ac6a7264413a6816379d97e89c0 2024-11-13T18:38:41,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/27869b390ec348f0924b70ea277181b0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/27869b390ec348f0924b70ea277181b0 2024-11-13T18:38:41,431 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/27869b390ec348f0924b70ea277181b0, entries=150, sequenceid=362, filesize=12.0 K 2024-11-13T18:38:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/82157507cd93412e8ceee03a4ca040ff as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/82157507cd93412e8ceee03a4ca040ff 2024-11-13T18:38:41,435 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/82157507cd93412e8ceee03a4ca040ff, entries=150, sequenceid=362, filesize=12.0 K 2024-11-13T18:38:41,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/b3108ac6a7264413a6816379d97e89c0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b3108ac6a7264413a6816379d97e89c0 2024-11-13T18:38:41,439 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b3108ac6a7264413a6816379d97e89c0, entries=150, sequenceid=362, filesize=12.0 K 2024-11-13T18:38:41,440 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 464ms, sequenceid=362, compaction requested=false 2024-11-13T18:38:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-13T18:38:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-13T18:38:41,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T18:38:41,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3860 sec 2024-11-13T18:38:41,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.3910 sec 2024-11-13T18:38:41,744 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/adc622c526454128ac0e37c9a01f8385 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/adc622c526454128ac0e37c9a01f8385 2024-11-13T18:38:41,750 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into adc622c526454128ac0e37c9a01f8385(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:41,750 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:41,750 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, priority=11, startTime=1731523120866; duration=0sec 2024-11-13T18:38:41,750 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:41,750 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C 2024-11-13T18:38:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:41,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:38:41,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ead7445f5e504d7d94a1dbe0c0a3faae is 50, key is test_row_0/A:col10/1731523121857/Put/seqid=0 2024-11-13T18:38:41,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742342_1518 (size=12301) 2024-11-13T18:38:41,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ead7445f5e504d7d94a1dbe0c0a3faae 2024-11-13T18:38:41,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/5d1dba629eed445e8e2d7b7c4b958864 is 50, key is test_row_0/B:col10/1731523121857/Put/seqid=0 2024-11-13T18:38:41,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742343_1519 (size=12301) 2024-11-13T18:38:41,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523181932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523181947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523181949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523182053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523182063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523182073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T18:38:42,204 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:42,204 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-13T18:38:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:38:42,206 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:42,207 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:42,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:42,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523182263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/5d1dba629eed445e8e2d7b7c4b958864 2024-11-13T18:38:42,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/3e96b059240841298fc71da0d80783cf is 50, key is test_row_0/C:col10/1731523121857/Put/seqid=0 2024-11-13T18:38:42,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523182283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523182294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742344_1520 (size=12301) 2024-11-13T18:38:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:38:42,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:38:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:38:42,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:42,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:38:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523182583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523182603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523182604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:42,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:38:42,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:42,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:42,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/3e96b059240841298fc71da0d80783cf 2024-11-13T18:38:42,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/ead7445f5e504d7d94a1dbe0c0a3faae as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ead7445f5e504d7d94a1dbe0c0a3faae 2024-11-13T18:38:42,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ead7445f5e504d7d94a1dbe0c0a3faae, entries=150, sequenceid=376, filesize=12.0 K 2024-11-13T18:38:42,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/5d1dba629eed445e8e2d7b7c4b958864 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5d1dba629eed445e8e2d7b7c4b958864 2024-11-13T18:38:42,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5d1dba629eed445e8e2d7b7c4b958864, entries=150, sequenceid=376, filesize=12.0 K 2024-11-13T18:38:42,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/3e96b059240841298fc71da0d80783cf as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/3e96b059240841298fc71da0d80783cf 2024-11-13T18:38:42,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/3e96b059240841298fc71da0d80783cf, entries=150, sequenceid=376, filesize=12.0 K 2024-11-13T18:38:42,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 853ms, sequenceid=376, compaction requested=true 2024-11-13T18:38:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:42,719 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:42,719 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:42,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:42,720 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:42,720 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A is initiating minor compaction (all files) 2024-11-13T18:38:42,720 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,720 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/564062c5e9ad401eac1cad8bd2c33752, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/27869b390ec348f0924b70ea277181b0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ead7445f5e504d7d94a1dbe0c0a3faae] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=36.8 K 2024-11-13T18:38:42,720 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:42,720 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B is initiating minor compaction (all files) 2024-11-13T18:38:42,720 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,720 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 564062c5e9ad401eac1cad8bd2c33752, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731523119216 2024-11-13T18:38:42,720 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ba55587565904131b99eb757cbec7aff, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/82157507cd93412e8ceee03a4ca040ff, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5d1dba629eed445e8e2d7b7c4b958864] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=36.8 K 2024-11-13T18:38:42,721 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27869b390ec348f0924b70ea277181b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731523119619 2024-11-13T18:38:42,721 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ba55587565904131b99eb757cbec7aff, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731523119216 2024-11-13T18:38:42,721 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 82157507cd93412e8ceee03a4ca040ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731523119619 2024-11-13T18:38:42,721 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ead7445f5e504d7d94a1dbe0c0a3faae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731523121849 2024-11-13T18:38:42,721 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1dba629eed445e8e2d7b7c4b958864, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731523121849 2024-11-13T18:38:42,729 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#A#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:42,730 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/10cde000aa6e4ab2b82a26a13afef3be is 50, key is test_row_0/A:col10/1731523121857/Put/seqid=0 2024-11-13T18:38:42,732 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:42,732 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/5a36ad4293ff4b6287b70c87d2fa6ebe is 50, key is test_row_0/B:col10/1731523121857/Put/seqid=0 2024-11-13T18:38:42,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742345_1521 (size=13187) 2024-11-13T18:38:42,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742346_1522 (size=13187) 2024-11-13T18:38:42,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T18:38:42,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:42,818 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:38:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/efb06b57d965424cbb685c3919c3d5ae is 50, key is test_row_0/A:col10/1731523121929/Put/seqid=0 2024-11-13T18:38:42,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742347_1523 (size=12301) 2024-11-13T18:38:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:38:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. as already flushing 2024-11-13T18:38:43,147 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/10cde000aa6e4ab2b82a26a13afef3be as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/10cde000aa6e4ab2b82a26a13afef3be 2024-11-13T18:38:43,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523183135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,151 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 10cde000aa6e4ab2b82a26a13afef3be(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:43,151 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:43,151 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, priority=13, startTime=1731523122719; duration=0sec 2024-11-13T18:38:43,151 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:43,151 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A 2024-11-13T18:38:43,151 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:43,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,152 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523183136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,152 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C is initiating minor compaction (all files) 2024-11-13T18:38:43,152 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:43,152 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/adc622c526454128ac0e37c9a01f8385, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b3108ac6a7264413a6816379d97e89c0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/3e96b059240841298fc71da0d80783cf] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=36.8 K 2024-11-13T18:38:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,153 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting adc622c526454128ac0e37c9a01f8385, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731523119216 2024-11-13T18:38:43,153 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3108ac6a7264413a6816379d97e89c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731523119619 2024-11-13T18:38:43,153 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e96b059240841298fc71da0d80783cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731523121849 2024-11-13T18:38:43,158 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/5a36ad4293ff4b6287b70c87d2fa6ebe as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5a36ad4293ff4b6287b70c87d2fa6ebe 2024-11-13T18:38:43,162 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 5a36ad4293ff4b6287b70c87d2fa6ebe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:43,162 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:43,162 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, priority=13, startTime=1731523122719; duration=0sec 2024-11-13T18:38:43,162 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:43,162 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B 2024-11-13T18:38:43,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523183150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,164 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:43,164 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/b844555676a244ceac736fb014ed58ec is 50, key is test_row_0/C:col10/1731523121857/Put/seqid=0 2024-11-13T18:38:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742348_1524 (size=13187) 2024-11-13T18:38:43,176 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/b844555676a244ceac736fb014ed58ec as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b844555676a244ceac736fb014ed58ec 2024-11-13T18:38:43,180 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into b844555676a244ceac736fb014ed58ec(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:43,180 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:43,180 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, priority=13, startTime=1731523122719; duration=0sec 2024-11-13T18:38:43,180 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:43,180 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C 2024-11-13T18:38:43,227 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/efb06b57d965424cbb685c3919c3d5ae 2024-11-13T18:38:43,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2 is 50, key is test_row_0/B:col10/1731523121929/Put/seqid=0 2024-11-13T18:38:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742349_1525 (size=12301) 2024-11-13T18:38:43,241 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2 2024-11-13T18:38:43,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ac438725f74d461c9c12af850953066e is 50, key is test_row_0/C:col10/1731523121929/Put/seqid=0 2024-11-13T18:38:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742350_1526 (size=12301) 2024-11-13T18:38:43,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523183253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523183263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523183273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:38:43,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523183463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523183483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523183493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,656 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ac438725f74d461c9c12af850953066e 2024-11-13T18:38:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/efb06b57d965424cbb685c3919c3d5ae as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/efb06b57d965424cbb685c3919c3d5ae 2024-11-13T18:38:43,663 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/efb06b57d965424cbb685c3919c3d5ae, entries=150, sequenceid=401, filesize=12.0 K 2024-11-13T18:38:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2 2024-11-13T18:38:43,667 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2, entries=150, sequenceid=401, filesize=12.0 K 2024-11-13T18:38:43,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/ac438725f74d461c9c12af850953066e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ac438725f74d461c9c12af850953066e 2024-11-13T18:38:43,671 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ac438725f74d461c9c12af850953066e, entries=150, sequenceid=401, filesize=12.0 K 2024-11-13T18:38:43,671 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 853ms, sequenceid=401, compaction requested=false 2024-11-13T18:38:43,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:43,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:43,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-13T18:38:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-13T18:38:43,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T18:38:43,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-11-13T18:38:43,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4750 sec 2024-11-13T18:38:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:43,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:43,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:38:43,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/17a4add72ebf42c3b12c46a2d5871c88 is 50, key is test_row_0/A:col10/1731523123135/Put/seqid=0 2024-11-13T18:38:43,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742351_1527 (size=14741) 2024-11-13T18:38:43,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523183873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,883 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:38:43,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523183874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523183876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:43,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523183983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523183994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523183994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,029 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-11-13T18:38:44,029 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:44,029 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,029 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:44,032 INFO [Thread-2013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2013. 2024-11-13T18:38:44,032 DEBUG [Thread-2013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:44,032 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,032 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:44,040 INFO [Thread-2009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2009. 2024-11-13T18:38:44,040 DEBUG [Thread-2009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:44,040 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,040 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:44,042 INFO [Thread-2011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2011. 2024-11-13T18:38:44,042 DEBUG [Thread-2011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:44,042 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,042 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:44,044 INFO [Thread-2007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2007. 2024-11-13T18:38:44,044 DEBUG [Thread-2007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:44,044 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,044 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:44,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:44,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/17a4add72ebf42c3b12c46a2d5871c88 2024-11-13T18:38:44,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:44,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:44,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523184203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523184203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523184203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/9e7336ac18664c79b8c70c9ce17f5199 is 50, key is test_row_0/B:col10/1731523123135/Put/seqid=0 2024-11-13T18:38:44,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742352_1528 (size=12301) 2024-11-13T18:38:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T18:38:44,354 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523184514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523184514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523184514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:44,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/9e7336ac18664c79b8c70c9ce17f5199 2024-11-13T18:38:44,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1dd90401fb234594b34dd3698b9f8a11 is 50, key is test_row_0/C:col10/1731523123135/Put/seqid=0 2024-11-13T18:38:44,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742353_1529 (size=12301) 2024-11-13T18:38:45,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1dd90401fb234594b34dd3698b9f8a11 2024-11-13T18:38:45,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/17a4add72ebf42c3b12c46a2d5871c88 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/17a4add72ebf42c3b12c46a2d5871c88 2024-11-13T18:38:45,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:45,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43626 deadline: 1731523185023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:45,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1731523185023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1731523185023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 , the old value is region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., hostname=39e84130bbc9,35235,1731522975587, seqNum=2 because the exception is null or not the one we care about 2024-11-13T18:38:45,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/17a4add72ebf42c3b12c46a2d5871c88, entries=200, sequenceid=416, filesize=14.4 K 2024-11-13T18:38:45,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/9e7336ac18664c79b8c70c9ce17f5199 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9e7336ac18664c79b8c70c9ce17f5199 2024-11-13T18:38:45,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9e7336ac18664c79b8c70c9ce17f5199, entries=150, sequenceid=416, filesize=12.0 K 2024-11-13T18:38:45,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/1dd90401fb234594b34dd3698b9f8a11 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1dd90401fb234594b34dd3698b9f8a11 2024-11-13T18:38:45,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1dd90401fb234594b34dd3698b9f8a11, entries=150, sequenceid=416, filesize=12.0 K 2024-11-13T18:38:45,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 1243ms, sequenceid=416, compaction requested=true 2024-11-13T18:38:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:45,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:45,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:45,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:45,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:45,033 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:45,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:45,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:45,033 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:45,033 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:45,033 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:45,033 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A is initiating minor compaction (all files) 2024-11-13T18:38:45,033 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B is initiating minor compaction (all files) 2024-11-13T18:38:45,033 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:45,033 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:45,033 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/10cde000aa6e4ab2b82a26a13afef3be, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/efb06b57d965424cbb685c3919c3d5ae, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/17a4add72ebf42c3b12c46a2d5871c88] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=39.3 K 2024-11-13T18:38:45,033 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5a36ad4293ff4b6287b70c87d2fa6ebe, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9e7336ac18664c79b8c70c9ce17f5199] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=36.9 K 2024-11-13T18:38:45,034 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10cde000aa6e4ab2b82a26a13afef3be, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731523121849 2024-11-13T18:38:45,034 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a36ad4293ff4b6287b70c87d2fa6ebe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731523121849 2024-11-13T18:38:45,034 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb06b57d965424cbb685c3919c3d5ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731523121929 2024-11-13T18:38:45,034 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8604c3fe3e1c4b97b2d58e3d0ea9acc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731523121929 2024-11-13T18:38:45,034 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a4add72ebf42c3b12c46a2d5871c88, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731523123134 2024-11-13T18:38:45,034 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7336ac18664c79b8c70c9ce17f5199, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731523123135 2024-11-13T18:38:45,039 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:45,039 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#A#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:45,040 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/cb2b7c290bf9481faff2015facf92272 is 50, key is test_row_0/B:col10/1731523123135/Put/seqid=0 2024-11-13T18:38:45,040 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/47599f05fa654665b4b887b7ce0eeb03 is 50, key is test_row_0/A:col10/1731523123135/Put/seqid=0 2024-11-13T18:38:45,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742354_1530 (size=13289) 2024-11-13T18:38:45,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742355_1531 (size=13289) 2024-11-13T18:38:45,447 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/47599f05fa654665b4b887b7ce0eeb03 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47599f05fa654665b4b887b7ce0eeb03 2024-11-13T18:38:45,447 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/cb2b7c290bf9481faff2015facf92272 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/cb2b7c290bf9481faff2015facf92272 2024-11-13T18:38:45,450 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into cb2b7c290bf9481faff2015facf92272(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:45,450 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:45,450 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, priority=13, startTime=1731523125033; duration=0sec 2024-11-13T18:38:45,451 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 47599f05fa654665b4b887b7ce0eeb03(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:45,451 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:45,451 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:45,451 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:B 2024-11-13T18:38:45,451 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, priority=13, startTime=1731523125032; duration=0sec 2024-11-13T18:38:45,451 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:45,451 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:45,451 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:A 2024-11-13T18:38:45,451 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:45,451 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C is initiating minor compaction (all files) 2024-11-13T18:38:45,451 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C in TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:45,451 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b844555676a244ceac736fb014ed58ec, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ac438725f74d461c9c12af850953066e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1dd90401fb234594b34dd3698b9f8a11] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp, totalSize=36.9 K 2024-11-13T18:38:45,452 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b844555676a244ceac736fb014ed58ec, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731523121849 2024-11-13T18:38:45,452 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ac438725f74d461c9c12af850953066e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731523121929 2024-11-13T18:38:45,452 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd90401fb234594b34dd3698b9f8a11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731523123135 2024-11-13T18:38:45,457 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:45,458 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/9bbc0938e5364184948b6e2f48ec3002 is 50, key is test_row_0/C:col10/1731523123135/Put/seqid=0 2024-11-13T18:38:45,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742356_1532 (size=13289) 2024-11-13T18:38:45,465 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/9bbc0938e5364184948b6e2f48ec3002 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/9bbc0938e5364184948b6e2f48ec3002 2024-11-13T18:38:45,469 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C of 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 into 9bbc0938e5364184948b6e2f48ec3002(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:45,469 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:45,469 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., storeName=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, priority=13, startTime=1731523125033; duration=0sec 2024-11-13T18:38:45,469 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:45,469 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166:C 2024-11-13T18:38:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:46,035 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:46,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T18:38:46,035 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-11-13T18:38:46,035 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-13T18:38:46,035 INFO [Thread-1996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1996. 2024-11-13T18:38:46,035 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:46,035 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:46,035 DEBUG [Thread-1996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:46,035 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,035 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,035 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,035 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,035 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,035 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:46,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:46,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:46,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/0a99353559fd458a9c6b11d59cff468c is 50, key is test_row_0/A:col10/1731523126033/Put/seqid=0 2024-11-13T18:38:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742357_1533 (size=12301) 2024-11-13T18:38:46,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/0a99353559fd458a9c6b11d59cff468c 2024-11-13T18:38:46,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/e5ee4ab41e1149ab8b5acbbe16dff600 is 50, key is test_row_0/B:col10/1731523126033/Put/seqid=0 2024-11-13T18:38:46,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742358_1534 (size=12301) 2024-11-13T18:38:46,735 INFO [Thread-1998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1998. 2024-11-13T18:38:46,735 INFO [Thread-2002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2002. 2024-11-13T18:38:46,735 DEBUG [Thread-2002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:46,735 DEBUG [Thread-1998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:38:46,735 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,735 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,736 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,736 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:46,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 956 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2868 rows 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 962 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2886 rows 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 956 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2868 rows 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 954 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2862 rows 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 953 2024-11-13T18:38:46,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2859 rows 2024-11-13T18:38:46,736 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:38:46,736 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:38:46,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:46,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:38:46,737 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:38:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:38:46,740 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523126740"}]},"ts":"1731523126740"} 2024-11-13T18:38:46,741 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:38:46,741 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:38:46,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:38:46,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, UNASSIGN}] 2024-11-13T18:38:46,744 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, UNASSIGN 2024-11-13T18:38:46,744 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:46,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, UNASSIGN because future has completed 2024-11-13T18:38:46,745 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:38:46,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:38:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:38:46,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/e5ee4ab41e1149ab8b5acbbe16dff600 2024-11-13T18:38:46,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a544243b84bc471dafe4184cd999b264 is 50, key is test_row_0/C:col10/1731523126033/Put/seqid=0 2024-11-13T18:38:46,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742359_1535 (size=12301) 2024-11-13T18:38:46,897 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:46,897 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:38:46,897 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, disabling compactions & flushes 2024-11-13T18:38:46,897 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:38:47,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a544243b84bc471dafe4184cd999b264 2024-11-13T18:38:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/0a99353559fd458a9c6b11d59cff468c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/0a99353559fd458a9c6b11d59cff468c 2024-11-13T18:38:47,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/0a99353559fd458a9c6b11d59cff468c, entries=150, sequenceid=446, filesize=12.0 K 2024-11-13T18:38:47,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/e5ee4ab41e1149ab8b5acbbe16dff600 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e5ee4ab41e1149ab8b5acbbe16dff600 2024-11-13T18:38:47,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e5ee4ab41e1149ab8b5acbbe16dff600, entries=150, sequenceid=446, filesize=12.0 K 2024-11-13T18:38:47,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/a544243b84bc471dafe4184cd999b264 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a544243b84bc471dafe4184cd999b264 2024-11-13T18:38:47,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a544243b84bc471dafe4184cd999b264, entries=150, sequenceid=446, filesize=12.0 K 2024-11-13T18:38:47,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=13.42 KB/13740 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 1238ms, sequenceid=446, compaction requested=false 2024-11-13T18:38:47,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: 2024-11-13T18:38:47,274 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:47,274 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:47,274 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. after waiting 0 ms 2024-11-13T18:38:47,274 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:47,274 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-13T18:38:47,277 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/b7bd5e7fbd924684aeb1f1b624361c60 is 50, key is test_row_0/A:col10/1731523126734/Put/seqid=0 2024-11-13T18:38:47,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742360_1536 (size=9857) 2024-11-13T18:38:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:38:47,680 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/b7bd5e7fbd924684aeb1f1b624361c60 2024-11-13T18:38:47,686 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7d666ec0b5664b0882e18347b0f87e69 is 50, key is test_row_0/B:col10/1731523126734/Put/seqid=0 2024-11-13T18:38:47,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742361_1537 (size=9857) 2024-11-13T18:38:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:38:48,089 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7d666ec0b5664b0882e18347b0f87e69 2024-11-13T18:38:48,094 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/9ec826277f774920a24408bee4b6c116 is 50, key is test_row_0/C:col10/1731523126734/Put/seqid=0 2024-11-13T18:38:48,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742362_1538 (size=9857) 2024-11-13T18:38:48,497 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/9ec826277f774920a24408bee4b6c116 2024-11-13T18:38:48,500 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/A/b7bd5e7fbd924684aeb1f1b624361c60 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/b7bd5e7fbd924684aeb1f1b624361c60 2024-11-13T18:38:48,503 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/b7bd5e7fbd924684aeb1f1b624361c60, entries=100, sequenceid=451, filesize=9.6 K 2024-11-13T18:38:48,504 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/B/7d666ec0b5664b0882e18347b0f87e69 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7d666ec0b5664b0882e18347b0f87e69 2024-11-13T18:38:48,506 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7d666ec0b5664b0882e18347b0f87e69, entries=100, sequenceid=451, filesize=9.6 K 2024-11-13T18:38:48,507 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/.tmp/C/9ec826277f774920a24408bee4b6c116 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/9ec826277f774920a24408bee4b6c116 2024-11-13T18:38:48,509 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/9ec826277f774920a24408bee4b6c116, entries=100, sequenceid=451, filesize=9.6 K 2024-11-13T18:38:48,510 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 1236ms, sequenceid=451, compaction requested=true 2024-11-13T18:38:48,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/f34b98d46c7041ed908c18b08a3594d3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6c060f6b1e854a85bebfa054f07ba7be, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a111a6d381e74e26b012143115d75020, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cc3222dbd5ab49249ed35bd274f6b1b3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea6e3ee265954d35a44b1e8a2b44bdb7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7c8b193c26ff4e669ef33d632ba39b13, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/00218c3f044f4d3e95a7c7081be6a949, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cf220dcbe24a45a9b11a0736ef0acb72, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/9f5c4895ee0c41c091e4a072ead101d0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/bb3607d8d8a44b788600c2e0113f1c5b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6b74219b814b4b7996f3c56c4c9d1bdb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/c1b4ab52c8b64326a80f44d1fe7336e7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/19353a30bdce43c1834a10d01b5740fc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/3d60a8fe060644fd806351b05be40043, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/df82440f6f614788ae68272445f937a3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7ccc7ede5d0e46f1b4cab3df14e452c4, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea0a07a5fbe24582999e9b6078a0c242, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47e4a52f694449ad99cea3c299c88640, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/4a380823cb8b4f43b124682e6b9a9bed, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/996f325569ec434dad1ee13ff8d99b97, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a2861409b1fb4bf58a930253e1a05994, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/24828cdcfe574be7becc76c1cff37892, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/5cbdbf56ab074f4a84ff3c1e9ace0b86, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/564062c5e9ad401eac1cad8bd2c33752, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/27869b390ec348f0924b70ea277181b0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/10cde000aa6e4ab2b82a26a13afef3be, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ead7445f5e504d7d94a1dbe0c0a3faae, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/efb06b57d965424cbb685c3919c3d5ae, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/17a4add72ebf42c3b12c46a2d5871c88] to archive 2024-11-13T18:38:48,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:48,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/f34b98d46c7041ed908c18b08a3594d3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/f34b98d46c7041ed908c18b08a3594d3 2024-11-13T18:38:48,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6c060f6b1e854a85bebfa054f07ba7be to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6c060f6b1e854a85bebfa054f07ba7be 2024-11-13T18:38:48,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a111a6d381e74e26b012143115d75020 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a111a6d381e74e26b012143115d75020 2024-11-13T18:38:48,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cc3222dbd5ab49249ed35bd274f6b1b3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cc3222dbd5ab49249ed35bd274f6b1b3 2024-11-13T18:38:48,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea6e3ee265954d35a44b1e8a2b44bdb7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea6e3ee265954d35a44b1e8a2b44bdb7 2024-11-13T18:38:48,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7c8b193c26ff4e669ef33d632ba39b13 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7c8b193c26ff4e669ef33d632ba39b13 2024-11-13T18:38:48,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/00218c3f044f4d3e95a7c7081be6a949 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/00218c3f044f4d3e95a7c7081be6a949 2024-11-13T18:38:48,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cf220dcbe24a45a9b11a0736ef0acb72 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/cf220dcbe24a45a9b11a0736ef0acb72 2024-11-13T18:38:48,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/9f5c4895ee0c41c091e4a072ead101d0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/9f5c4895ee0c41c091e4a072ead101d0 2024-11-13T18:38:48,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/bb3607d8d8a44b788600c2e0113f1c5b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/bb3607d8d8a44b788600c2e0113f1c5b 2024-11-13T18:38:48,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6b74219b814b4b7996f3c56c4c9d1bdb to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/6b74219b814b4b7996f3c56c4c9d1bdb 2024-11-13T18:38:48,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/c1b4ab52c8b64326a80f44d1fe7336e7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/c1b4ab52c8b64326a80f44d1fe7336e7 2024-11-13T18:38:48,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/19353a30bdce43c1834a10d01b5740fc to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/19353a30bdce43c1834a10d01b5740fc 2024-11-13T18:38:48,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/3d60a8fe060644fd806351b05be40043 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/3d60a8fe060644fd806351b05be40043 2024-11-13T18:38:48,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/df82440f6f614788ae68272445f937a3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/df82440f6f614788ae68272445f937a3 2024-11-13T18:38:48,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7ccc7ede5d0e46f1b4cab3df14e452c4 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/7ccc7ede5d0e46f1b4cab3df14e452c4 2024-11-13T18:38:48,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea0a07a5fbe24582999e9b6078a0c242 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ea0a07a5fbe24582999e9b6078a0c242 2024-11-13T18:38:48,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47e4a52f694449ad99cea3c299c88640 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47e4a52f694449ad99cea3c299c88640 2024-11-13T18:38:48,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/4a380823cb8b4f43b124682e6b9a9bed to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/4a380823cb8b4f43b124682e6b9a9bed 2024-11-13T18:38:48,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/996f325569ec434dad1ee13ff8d99b97 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/996f325569ec434dad1ee13ff8d99b97 2024-11-13T18:38:48,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a2861409b1fb4bf58a930253e1a05994 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/a2861409b1fb4bf58a930253e1a05994 2024-11-13T18:38:48,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/24828cdcfe574be7becc76c1cff37892 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/24828cdcfe574be7becc76c1cff37892 2024-11-13T18:38:48,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/5cbdbf56ab074f4a84ff3c1e9ace0b86 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/5cbdbf56ab074f4a84ff3c1e9ace0b86 2024-11-13T18:38:48,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/564062c5e9ad401eac1cad8bd2c33752 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/564062c5e9ad401eac1cad8bd2c33752 2024-11-13T18:38:48,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/27869b390ec348f0924b70ea277181b0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/27869b390ec348f0924b70ea277181b0 2024-11-13T18:38:48,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/10cde000aa6e4ab2b82a26a13afef3be to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/10cde000aa6e4ab2b82a26a13afef3be 2024-11-13T18:38:48,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ead7445f5e504d7d94a1dbe0c0a3faae to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/ead7445f5e504d7d94a1dbe0c0a3faae 2024-11-13T18:38:48,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/efb06b57d965424cbb685c3919c3d5ae to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/efb06b57d965424cbb685c3919c3d5ae 2024-11-13T18:38:48,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/17a4add72ebf42c3b12c46a2d5871c88 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/17a4add72ebf42c3b12c46a2d5871c88 2024-11-13T18:38:48,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/417e9a6a1dfd4a84bab31a44f4a44dc3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c40cd6d049344890bc29f01834934684, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e4a407508b9a4986b508da9ed26c6e45, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/d193baa63aac41fcb043d56e21e409ce, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a11ab326a97044c78131e2bb82cb6179, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a8da9a759a834755a92c2a5659a78913, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c54eddebf2094229911cdd338df552cd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/58eff25a72af4129956142a7920f302c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2ac3c0506a984716baa6c775f83947a1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a516b3d988764cb2bcb863456c1e39cf, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7472528352c249bcaa32a9e1e406197e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9d78004d7a484593921adafd51215191, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7c2ab8c934104245b1700b69ecbb26de, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/b3fb99bd4cf64b4393972ed6eeb8c2ca, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/17e25aafaf72417497cd55a229d45509, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ade82f0cce9d450e8e32634fb60c7152, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/340e98c20ab34d53a73e574922859947, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7035eec604f94cc090630b284481545b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/63efc8fc342c43f8bbae5531c8426ade, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/0ea4e5473e3f49cebca8b9771fb0bfba, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8cd389a16e244b2eb74a3fbbe1cb3269, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2bf9d72df863401ba8442fd4e73ba0bd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ba55587565904131b99eb757cbec7aff, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/200b22ca7dfa4598820167f720beb410, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/82157507cd93412e8ceee03a4ca040ff, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5a36ad4293ff4b6287b70c87d2fa6ebe, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5d1dba629eed445e8e2d7b7c4b958864, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9e7336ac18664c79b8c70c9ce17f5199] to archive 2024-11-13T18:38:48,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:48,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/417e9a6a1dfd4a84bab31a44f4a44dc3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/417e9a6a1dfd4a84bab31a44f4a44dc3 2024-11-13T18:38:48,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c40cd6d049344890bc29f01834934684 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c40cd6d049344890bc29f01834934684 2024-11-13T18:38:48,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e4a407508b9a4986b508da9ed26c6e45 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e4a407508b9a4986b508da9ed26c6e45 2024-11-13T18:38:48,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/d193baa63aac41fcb043d56e21e409ce to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/d193baa63aac41fcb043d56e21e409ce 2024-11-13T18:38:48,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a11ab326a97044c78131e2bb82cb6179 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a11ab326a97044c78131e2bb82cb6179 2024-11-13T18:38:48,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a8da9a759a834755a92c2a5659a78913 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a8da9a759a834755a92c2a5659a78913 2024-11-13T18:38:48,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c54eddebf2094229911cdd338df552cd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/c54eddebf2094229911cdd338df552cd 2024-11-13T18:38:48,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/58eff25a72af4129956142a7920f302c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/58eff25a72af4129956142a7920f302c 2024-11-13T18:38:48,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2ac3c0506a984716baa6c775f83947a1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2ac3c0506a984716baa6c775f83947a1 2024-11-13T18:38:48,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a516b3d988764cb2bcb863456c1e39cf to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/a516b3d988764cb2bcb863456c1e39cf 2024-11-13T18:38:48,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7472528352c249bcaa32a9e1e406197e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7472528352c249bcaa32a9e1e406197e 2024-11-13T18:38:48,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9d78004d7a484593921adafd51215191 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9d78004d7a484593921adafd51215191 2024-11-13T18:38:48,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7c2ab8c934104245b1700b69ecbb26de to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7c2ab8c934104245b1700b69ecbb26de 2024-11-13T18:38:48,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/b3fb99bd4cf64b4393972ed6eeb8c2ca to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/b3fb99bd4cf64b4393972ed6eeb8c2ca 2024-11-13T18:38:48,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/17e25aafaf72417497cd55a229d45509 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/17e25aafaf72417497cd55a229d45509 2024-11-13T18:38:48,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ade82f0cce9d450e8e32634fb60c7152 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ade82f0cce9d450e8e32634fb60c7152 2024-11-13T18:38:48,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/340e98c20ab34d53a73e574922859947 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/340e98c20ab34d53a73e574922859947 2024-11-13T18:38:48,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7035eec604f94cc090630b284481545b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7035eec604f94cc090630b284481545b 2024-11-13T18:38:48,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/63efc8fc342c43f8bbae5531c8426ade to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/63efc8fc342c43f8bbae5531c8426ade 2024-11-13T18:38:48,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/0ea4e5473e3f49cebca8b9771fb0bfba to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/0ea4e5473e3f49cebca8b9771fb0bfba 2024-11-13T18:38:48,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8cd389a16e244b2eb74a3fbbe1cb3269 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8cd389a16e244b2eb74a3fbbe1cb3269 2024-11-13T18:38:48,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2bf9d72df863401ba8442fd4e73ba0bd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/2bf9d72df863401ba8442fd4e73ba0bd 2024-11-13T18:38:48,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ba55587565904131b99eb757cbec7aff to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/ba55587565904131b99eb757cbec7aff 2024-11-13T18:38:48,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/200b22ca7dfa4598820167f720beb410 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/200b22ca7dfa4598820167f720beb410 2024-11-13T18:38:48,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/82157507cd93412e8ceee03a4ca040ff to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/82157507cd93412e8ceee03a4ca040ff 2024-11-13T18:38:48,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5a36ad4293ff4b6287b70c87d2fa6ebe to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5a36ad4293ff4b6287b70c87d2fa6ebe 2024-11-13T18:38:48,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5d1dba629eed445e8e2d7b7c4b958864 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/5d1dba629eed445e8e2d7b7c4b958864 2024-11-13T18:38:48,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/8604c3fe3e1c4b97b2d58e3d0ea9acc2 2024-11-13T18:38:48,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9e7336ac18664c79b8c70c9ce17f5199 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/9e7336ac18664c79b8c70c9ce17f5199 2024-11-13T18:38:48,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/67ff827c56544a28a1511fbd7c69ce28, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/095aa0b588d1482abf7e5978ecd41d77, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0cfe3a9dcfc74a828a7cd63b9684b0ec, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ad2be57fa2f941ddbb72f453350f96eb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a878637ab6c54af89455cd65b61fdc6f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/abffb6d0db3d4ddaa0cdc3bd5e435ca5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/84e20aeae908429ab85b3e7ed608d93c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/860551d749af46c890c3b034898fafad, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5ece69bde1e49d88431f101518e881c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/32567ad08e6c4b64a0d641d016348842, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a7f30228d01e4ff1a4eae9b468c242d1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ef647031420144138f584e83e43a3045, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/4ff274c1aa3f470ea7492bbe04d6d189, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5c4b6bc8a974fa784592cec80150fc2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/677dc7e021074d009ed4969b5669d817, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/33da2cf8dc7e49b1b0b662f4196c64ff, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ebe827a01c3149c6b7d365386ba5cb5e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ba6b2f0df19f48d3a964590a6c1e9b9f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0abccc05161747eabec1277cd166f221, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1cbd39c3f54b46698ea7be644dd71402, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/bc765f6e92144629b2f4d54fdac0f56d, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/2a8545889d7143daaa02e713ed7092f8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/adc622c526454128ac0e37c9a01f8385, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1b71bc16499c442fa616242debacc21f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b3108ac6a7264413a6816379d97e89c0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b844555676a244ceac736fb014ed58ec, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/3e96b059240841298fc71da0d80783cf, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ac438725f74d461c9c12af850953066e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1dd90401fb234594b34dd3698b9f8a11] to archive 2024-11-13T18:38:48,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:38:48,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/67ff827c56544a28a1511fbd7c69ce28 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/67ff827c56544a28a1511fbd7c69ce28 2024-11-13T18:38:48,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/095aa0b588d1482abf7e5978ecd41d77 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/095aa0b588d1482abf7e5978ecd41d77 2024-11-13T18:38:48,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0cfe3a9dcfc74a828a7cd63b9684b0ec to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0cfe3a9dcfc74a828a7cd63b9684b0ec 2024-11-13T18:38:48,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ad2be57fa2f941ddbb72f453350f96eb to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ad2be57fa2f941ddbb72f453350f96eb 2024-11-13T18:38:48,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a878637ab6c54af89455cd65b61fdc6f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a878637ab6c54af89455cd65b61fdc6f 2024-11-13T18:38:48,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/abffb6d0db3d4ddaa0cdc3bd5e435ca5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/abffb6d0db3d4ddaa0cdc3bd5e435ca5 2024-11-13T18:38:48,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/84e20aeae908429ab85b3e7ed608d93c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/84e20aeae908429ab85b3e7ed608d93c 2024-11-13T18:38:48,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/860551d749af46c890c3b034898fafad to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/860551d749af46c890c3b034898fafad 2024-11-13T18:38:48,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5ece69bde1e49d88431f101518e881c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5ece69bde1e49d88431f101518e881c 2024-11-13T18:38:48,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/32567ad08e6c4b64a0d641d016348842 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/32567ad08e6c4b64a0d641d016348842 2024-11-13T18:38:48,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a7f30228d01e4ff1a4eae9b468c242d1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a7f30228d01e4ff1a4eae9b468c242d1 2024-11-13T18:38:48,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ef647031420144138f584e83e43a3045 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ef647031420144138f584e83e43a3045 2024-11-13T18:38:48,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/4ff274c1aa3f470ea7492bbe04d6d189 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/4ff274c1aa3f470ea7492bbe04d6d189 2024-11-13T18:38:48,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5c4b6bc8a974fa784592cec80150fc2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/e5c4b6bc8a974fa784592cec80150fc2 2024-11-13T18:38:48,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/677dc7e021074d009ed4969b5669d817 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/677dc7e021074d009ed4969b5669d817 2024-11-13T18:38:48,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/33da2cf8dc7e49b1b0b662f4196c64ff to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/33da2cf8dc7e49b1b0b662f4196c64ff 2024-11-13T18:38:48,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ebe827a01c3149c6b7d365386ba5cb5e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ebe827a01c3149c6b7d365386ba5cb5e 2024-11-13T18:38:48,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ba6b2f0df19f48d3a964590a6c1e9b9f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ba6b2f0df19f48d3a964590a6c1e9b9f 2024-11-13T18:38:48,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0abccc05161747eabec1277cd166f221 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/0abccc05161747eabec1277cd166f221 2024-11-13T18:38:48,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1cbd39c3f54b46698ea7be644dd71402 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1cbd39c3f54b46698ea7be644dd71402 2024-11-13T18:38:48,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/bc765f6e92144629b2f4d54fdac0f56d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/bc765f6e92144629b2f4d54fdac0f56d 2024-11-13T18:38:48,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/2a8545889d7143daaa02e713ed7092f8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/2a8545889d7143daaa02e713ed7092f8 2024-11-13T18:38:48,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/adc622c526454128ac0e37c9a01f8385 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/adc622c526454128ac0e37c9a01f8385 2024-11-13T18:38:48,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1b71bc16499c442fa616242debacc21f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1b71bc16499c442fa616242debacc21f 2024-11-13T18:38:48,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b3108ac6a7264413a6816379d97e89c0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b3108ac6a7264413a6816379d97e89c0 2024-11-13T18:38:48,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b844555676a244ceac736fb014ed58ec to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/b844555676a244ceac736fb014ed58ec 2024-11-13T18:38:48,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/3e96b059240841298fc71da0d80783cf to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/3e96b059240841298fc71da0d80783cf 2024-11-13T18:38:48,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ac438725f74d461c9c12af850953066e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/ac438725f74d461c9c12af850953066e 2024-11-13T18:38:48,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1dd90401fb234594b34dd3698b9f8a11 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/1dd90401fb234594b34dd3698b9f8a11 2024-11-13T18:38:48,586 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/recovered.edits/454.seqid, newMaxSeqId=454, maxSeqId=1 2024-11-13T18:38:48,587 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. 2024-11-13T18:38:48,587 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166: Waiting for close lock at 1731523126897Running coprocessor pre-close hooks at 1731523126897Disabling compacts and flushes for region at 1731523126897Disabling writes for close at 1731523127274 (+377 ms)Obtaining lock to block concurrent updates at 1731523127274Preparing flush snapshotting stores in 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 at 1731523127274Finished memstore snapshotting TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731523127274Flushing stores of TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166. at 1731523127275 (+1 ms)Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A: creating writer at 1731523127275Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A: appending metadata at 1731523127277 (+2 ms)Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A: closing flushed file at 1731523127277Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B: creating writer at 1731523127683 (+406 ms)Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B: appending metadata at 1731523127685 (+2 ms)Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B: closing flushed file at 1731523127685Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C: creating writer at 1731523128092 (+407 ms)Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C: appending metadata at 1731523128093 (+1 ms)Flushing 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C: closing flushed file at 1731523128093Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e02048b: reopening flushed file at 1731523128500 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@574e5d37: reopening flushed file at 1731523128503 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23e18ae8: reopening flushed file at 1731523128506 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 in 1236ms, sequenceid=451, compaction requested=true at 1731523128510 (+4 ms)Writing region close event to WAL at 1731523128584 (+74 ms)Running coprocessor post-close hooks at 1731523128586 (+2 ms)Closed at 1731523128586 2024-11-13T18:38:48,588 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:48,589 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, regionState=CLOSED 2024-11-13T18:38:48,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:38:48,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-13T18:38:48,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, server=39e84130bbc9,35235,1731522975587 in 1.8460 sec 2024-11-13T18:38:48,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-13T18:38:48,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, UNASSIGN in 1.8500 sec 2024-11-13T18:38:48,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T18:38:48,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8530 sec 2024-11-13T18:38:48,598 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523128598"}]},"ts":"1731523128598"} 2024-11-13T18:38:48,599 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:38:48,599 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:38:48,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8630 sec 2024-11-13T18:38:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T18:38:48,884 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:48,885 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:38:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:48,886 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T18:38:48,887 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:48,888 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:48,890 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/recovered.edits] 2024-11-13T18:38:48,891 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/0a99353559fd458a9c6b11d59cff468c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/0a99353559fd458a9c6b11d59cff468c 2024-11-13T18:38:48,892 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47599f05fa654665b4b887b7ce0eeb03 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/47599f05fa654665b4b887b7ce0eeb03 2024-11-13T18:38:48,893 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/b7bd5e7fbd924684aeb1f1b624361c60 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/A/b7bd5e7fbd924684aeb1f1b624361c60 2024-11-13T18:38:48,894 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7d666ec0b5664b0882e18347b0f87e69 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/7d666ec0b5664b0882e18347b0f87e69 2024-11-13T18:38:48,895 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/cb2b7c290bf9481faff2015facf92272 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/cb2b7c290bf9481faff2015facf92272 2024-11-13T18:38:48,895 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e5ee4ab41e1149ab8b5acbbe16dff600 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/B/e5ee4ab41e1149ab8b5acbbe16dff600 2024-11-13T18:38:48,896 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/9bbc0938e5364184948b6e2f48ec3002 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/9bbc0938e5364184948b6e2f48ec3002 2024-11-13T18:38:48,897 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/9ec826277f774920a24408bee4b6c116 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/9ec826277f774920a24408bee4b6c116 2024-11-13T18:38:48,898 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a544243b84bc471dafe4184cd999b264 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/C/a544243b84bc471dafe4184cd999b264 2024-11-13T18:38:48,899 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/recovered.edits/454.seqid to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166/recovered.edits/454.seqid 2024-11-13T18:38:48,900 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/1e1cbe3c8a3f4b52fb2c2bfe3ffb9166 2024-11-13T18:38:48,900 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:38:48,901 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:48,903 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:38:48,904 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:38:48,905 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:48,905 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:38:48,905 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731523128905"}]},"ts":"9223372036854775807"} 2024-11-13T18:38:48,906 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:38:48,906 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1e1cbe3c8a3f4b52fb2c2bfe3ffb9166, NAME => 'TestAcidGuarantees,,1731523101826.1e1cbe3c8a3f4b52fb2c2bfe3ffb9166.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:38:48,906 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:38:48,907 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731523128907"}]},"ts":"9223372036854775807"} 2024-11-13T18:38:48,908 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:38:48,908 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:48,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 23 msec 2024-11-13T18:38:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T18:38:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:38:48,994 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:49,002 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242 (was 242), OpenFileDescriptor=457 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1069 (was 1355), ProcessCount=11 (was 11), AvailableMemoryMB=7764 (was 7878) 2024-11-13T18:38:49,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=1069, ProcessCount=11, AvailableMemoryMB=7764 2024-11-13T18:38:49,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:38:49,012 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:38:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:49,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:38:49,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-13T18:38:49,014 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:38:49,015 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:38:49,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742363_1539 (size=959) 2024-11-13T18:38:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:38:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:38:49,421 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462 2024-11-13T18:38:49,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742364_1540 (size=53) 2024-11-13T18:38:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:38:49,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:38:49,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 21be930938e864d73f0be2877df0a945, disabling compactions & flushes 2024-11-13T18:38:49,829 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:49,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:49,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. after waiting 0 ms 2024-11-13T18:38:49,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:49,829 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:49,829 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21be930938e864d73f0be2877df0a945: Waiting for close lock at 1731523129829Disabling compacts and flushes for region at 1731523129829Disabling writes for close at 1731523129829Writing region close event to WAL at 1731523129829Closed at 1731523129829 2024-11-13T18:38:49,830 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:38:49,830 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731523129830"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731523129830"}]},"ts":"1731523129830"} 2024-11-13T18:38:49,832 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:38:49,832 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:38:49,833 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523129832"}]},"ts":"1731523129832"} 2024-11-13T18:38:49,834 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T18:38:49,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, ASSIGN}] 2024-11-13T18:38:49,835 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, ASSIGN 2024-11-13T18:38:49,835 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, ASSIGN; state=OFFLINE, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=false 2024-11-13T18:38:49,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=21be930938e864d73f0be2877df0a945, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:49,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, ASSIGN because future has completed 2024-11-13T18:38:49,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:38:50,143 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:50,143 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:38:50,144 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,144 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:38:50,144 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,144 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,145 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,147 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21be930938e864d73f0be2877df0a945 columnFamilyName A 2024-11-13T18:38:50,147 DEBUG [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:50,147 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(327): Store=21be930938e864d73f0be2877df0a945/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:50,147 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,148 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21be930938e864d73f0be2877df0a945 columnFamilyName B 2024-11-13T18:38:50,148 DEBUG [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:50,148 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(327): Store=21be930938e864d73f0be2877df0a945/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:50,149 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,149 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21be930938e864d73f0be2877df0a945 columnFamilyName C 2024-11-13T18:38:50,150 DEBUG [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:50,150 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(327): Store=21be930938e864d73f0be2877df0a945/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:50,150 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,151 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,151 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,152 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,152 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,152 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:38:50,153 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:38:50,155 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:38:50,155 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 21be930938e864d73f0be2877df0a945; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59461172, jitterRate=-0.11395949125289917}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:38:50,155 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:50,155 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 21be930938e864d73f0be2877df0a945: Running coprocessor pre-open hook at 1731523130144Writing region info on filesystem at 1731523130144Initializing all the Stores at 1731523130145 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523130145Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523130145Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523130145Cleaning up temporary data from old regions at 1731523130152 (+7 ms)Running coprocessor post-open hooks at 1731523130155 (+3 ms)Region opened successfully at 1731523130155 2024-11-13T18:38:50,156 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., pid=140, masterSystemTime=1731523130140 2024-11-13T18:38:50,157 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:50,158 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:50,158 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=21be930938e864d73f0be2877df0a945, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:50,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:38:50,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-13T18:38:50,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 in 172 msec 2024-11-13T18:38:50,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-13T18:38:50,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, ASSIGN in 327 msec 2024-11-13T18:38:50,163 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:38:50,163 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523130163"}]},"ts":"1731523130163"} 2024-11-13T18:38:50,164 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T18:38:50,165 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:38:50,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-13T18:38:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T18:38:51,163 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:51,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:51,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:51,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:51,165 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6062603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:51,167 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b22037, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:51,167 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:51,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:51,170 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:38:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 39e84130bbc9,36105,1731522974847 2024-11-13T18:38:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7402e572 2024-11-13T18:38:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:38:51,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:38:51,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T18:38:51,172 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:38:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T18:38:51,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742365_1541 (size=995) 2024-11-13T18:38:51,583 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T18:38:51,583 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T18:38:51,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:38:51,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, REOPEN/MOVE}] 2024-11-13T18:38:51,588 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, REOPEN/MOVE 2024-11-13T18:38:51,588 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=21be930938e864d73f0be2877df0a945, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:51,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, REOPEN/MOVE because future has completed 2024-11-13T18:38:51,589 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:38:51,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:38:51,741 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:51,742 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:38:51,742 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 21be930938e864d73f0be2877df0a945, disabling compactions & flushes 2024-11-13T18:38:51,742 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:51,742 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:51,742 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. after waiting 0 ms 2024-11-13T18:38:51,742 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:51,745 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T18:38:51,745 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:51,745 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 21be930938e864d73f0be2877df0a945: Waiting for close lock at 1731523131742Running coprocessor pre-close hooks at 1731523131742Disabling compacts and flushes for region at 1731523131742Disabling writes for close at 1731523131742Writing region close event to WAL at 1731523131742Running coprocessor post-close hooks at 1731523131745 (+3 ms)Closed at 1731523131745 2024-11-13T18:38:51,745 WARN [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 21be930938e864d73f0be2877df0a945 to self. 2024-11-13T18:38:51,747 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:51,747 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=21be930938e864d73f0be2877df0a945, regionState=CLOSED 2024-11-13T18:38:51,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:38:51,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-13T18:38:51,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 in 160 msec 2024-11-13T18:38:51,751 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, REOPEN/MOVE; state=CLOSED, location=39e84130bbc9,35235,1731522975587; forceNewPlan=false, retain=true 2024-11-13T18:38:51,901 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=21be930938e864d73f0be2877df0a945, regionState=OPENING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:51,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, REOPEN/MOVE because future has completed 2024-11-13T18:38:51,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:38:52,057 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,057 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:38:52,057 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,057 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:38:52,057 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,057 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,059 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,059 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21be930938e864d73f0be2877df0a945 columnFamilyName A 2024-11-13T18:38:52,060 DEBUG [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:52,061 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(327): Store=21be930938e864d73f0be2877df0a945/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:52,061 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,062 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21be930938e864d73f0be2877df0a945 columnFamilyName B 2024-11-13T18:38:52,062 DEBUG [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:52,062 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(327): Store=21be930938e864d73f0be2877df0a945/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:52,062 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,063 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21be930938e864d73f0be2877df0a945 columnFamilyName C 2024-11-13T18:38:52,063 DEBUG [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:52,063 INFO [StoreOpener-21be930938e864d73f0be2877df0a945-1 {}] regionserver.HStore(327): Store=21be930938e864d73f0be2877df0a945/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:38:52,063 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,064 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,064 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,065 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,065 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,066 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:38:52,067 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,067 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 21be930938e864d73f0be2877df0a945; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66929196, jitterRate=-0.0026772618293762207}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:38:52,068 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,068 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 21be930938e864d73f0be2877df0a945: Running coprocessor pre-open hook at 1731523132058Writing region info on filesystem at 1731523132058Initializing all the Stores at 1731523132058Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523132058Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523132058Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731523132058Cleaning up temporary data from old regions at 1731523132065 (+7 ms)Running coprocessor post-open hooks at 1731523132068 (+3 ms)Region opened successfully at 1731523132068 2024-11-13T18:38:52,069 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., pid=145, masterSystemTime=1731523132054 2024-11-13T18:38:52,070 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,070 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,071 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=21be930938e864d73f0be2877df0a945, regionState=OPEN, openSeqNum=5, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:38:52,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-13T18:38:52,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 in 169 msec 2024-11-13T18:38:52,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-13T18:38:52,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, REOPEN/MOVE in 487 msec 2024-11-13T18:38:52,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-13T18:38:52,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-13T18:38:52,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-13T18:38:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T18:38:52,079 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:52,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,081 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77764ba8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,082 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444ca773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,084 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e1fa930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,085 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c8aa71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,086 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148a58bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f1420af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,089 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d462727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,090 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d3b315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,092 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d965d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,093 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4004a49d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,094 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bea369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,095 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28939e1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,097 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8acfd14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,098 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1241f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,099 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,099 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,100 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e67e504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,101 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c297e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,103 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb016e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,104 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93ff0d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,36105,-1 for getting cluster id 2024-11-13T18:38:52,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:38:52,105 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b58678c-3cab-43db-b775-68cd0f8afd7f' 2024-11-13T18:38:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:38:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b58678c-3cab-43db-b775-68cd0f8afd7f" 2024-11-13T18:38:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685affea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,36105,-1] 2024-11-13T18:38:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:38:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:38:52,106 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:38:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a08d0da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:38:52,107 DEBUG [Thread-2430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,107 DEBUG [Thread-2434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,107 DEBUG [Thread-2428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,108 DEBUG [Thread-2423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,108 DEBUG [Thread-2419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,108 DEBUG [Thread-2425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,109 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:52,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,109 DEBUG [Thread-2417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,109 DEBUG [Thread-2432 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,109 DEBUG [Thread-2421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-13T18:38:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,111 DEBUG [Thread-2436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:38:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T18:38:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,116 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,35235,1731522975587, seqNum=-1] 2024-11-13T18:38:52,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:38:52,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,119 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:52,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:38:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5] 2024-11-13T18:38:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,124 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:52,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:38:52,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523192149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132d5b34b8441d4f5b9fe11f7184f1ac9e_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523132124/Put/seqid=0 2024-11-13T18:38:52,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523192151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523192152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742366_1542 (size=12154) 2024-11-13T18:38:52,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523192153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:52,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523192154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132d5b34b8441d4f5b9fe11f7184f1ac9e_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d5b34b8441d4f5b9fe11f7184f1ac9e_21be930938e864d73f0be2877df0a945 2024-11-13T18:38:52,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/79f2fb258e664624a75a608af63417c9, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:52,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/79f2fb258e664624a75a608af63417c9 is 175, key is test_row_0/A:col10/1731523132124/Put/seqid=0 2024-11-13T18:38:52,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742367_1543 (size=30955) 2024-11-13T18:38:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T18:38:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523192263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523192264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523192264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523192264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523192264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T18:38:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T18:38:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T18:38:52,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523192473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523192473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523192473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523192474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523192483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/79f2fb258e664624a75a608af63417c9 2024-11-13T18:38:52,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T18:38:52,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:52,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/3b435627f8cb4ab1b53bca26062c9558 is 50, key is test_row_0/B:col10/1731523132124/Put/seqid=0 2024-11-13T18:38:52,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742368_1544 (size=12001) 2024-11-13T18:38:52,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/3b435627f8cb4ab1b53bca26062c9558 2024-11-13T18:38:52,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/2e216adf21d3470a89f4bea3e78e5319 is 50, key is test_row_0/C:col10/1731523132124/Put/seqid=0 2024-11-13T18:38:52,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742369_1545 (size=12001) 2024-11-13T18:38:52,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T18:38:52,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:52,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T18:38:52,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523192783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523192783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523192783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523192784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523192793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:52,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:52,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:52,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T18:38:52,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:52,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:52,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:53,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/2e216adf21d3470a89f4bea3e78e5319 2024-11-13T18:38:53,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/79f2fb258e664624a75a608af63417c9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f2fb258e664624a75a608af63417c9 2024-11-13T18:38:53,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f2fb258e664624a75a608af63417c9, entries=150, sequenceid=16, filesize=30.2 K 2024-11-13T18:38:53,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/3b435627f8cb4ab1b53bca26062c9558 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3b435627f8cb4ab1b53bca26062c9558 2024-11-13T18:38:53,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3b435627f8cb4ab1b53bca26062c9558, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T18:38:53,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T18:38:53,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:53,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:53,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:53,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/2e216adf21d3470a89f4bea3e78e5319 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/2e216adf21d3470a89f4bea3e78e5319 2024-11-13T18:38:53,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:53,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/2e216adf21d3470a89f4bea3e78e5319, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T18:38:53,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 21be930938e864d73f0be2877df0a945 in 921ms, sequenceid=16, compaction requested=false 2024-11-13T18:38:53,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:53,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T18:38:53,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:53,193 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:38:53,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130e0314d515af4fb8a9fd45c95b6e18b3_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523132151/Put/seqid=0 2024-11-13T18:38:53,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742370_1546 (size=12154) 2024-11-13T18:38:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T18:38:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:53,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:53,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523193303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523193303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523193304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523193305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523193305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523193413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523193413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523193413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523193413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:53,607 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130e0314d515af4fb8a9fd45c95b6e18b3_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130e0314d515af4fb8a9fd45c95b6e18b3_21be930938e864d73f0be2877df0a945 2024-11-13T18:38:53,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/386486a82a2849b4bebc8b501d660262, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:53,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/386486a82a2849b4bebc8b501d660262 is 175, key is test_row_0/A:col10/1731523132151/Put/seqid=0 2024-11-13T18:38:53,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742371_1547 (size=30955) 2024-11-13T18:38:53,612 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/386486a82a2849b4bebc8b501d660262 2024-11-13T18:38:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/b7e4c9526ada43d88950db2d422b60b9 is 50, key is test_row_0/B:col10/1731523132151/Put/seqid=0 2024-11-13T18:38:53,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523193623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523193623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523193623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523193624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742372_1548 (size=12001) 2024-11-13T18:38:53,634 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/b7e4c9526ada43d88950db2d422b60b9 2024-11-13T18:38:53,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e59d7c333b7348cb947360efb41450d6 is 50, key is test_row_0/C:col10/1731523132151/Put/seqid=0 2024-11-13T18:38:53,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742373_1549 (size=12001) 2024-11-13T18:38:53,659 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e59d7c333b7348cb947360efb41450d6 2024-11-13T18:38:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/386486a82a2849b4bebc8b501d660262 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/386486a82a2849b4bebc8b501d660262 2024-11-13T18:38:53,666 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/386486a82a2849b4bebc8b501d660262, entries=150, sequenceid=41, filesize=30.2 K 2024-11-13T18:38:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/b7e4c9526ada43d88950db2d422b60b9 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b7e4c9526ada43d88950db2d422b60b9 2024-11-13T18:38:53,670 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b7e4c9526ada43d88950db2d422b60b9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-13T18:38:53,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e59d7c333b7348cb947360efb41450d6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e59d7c333b7348cb947360efb41450d6 2024-11-13T18:38:53,675 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e59d7c333b7348cb947360efb41450d6, entries=150, sequenceid=41, filesize=11.7 K 2024-11-13T18:38:53,675 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 21be930938e864d73f0be2877df0a945 in 482ms, sequenceid=41, compaction requested=false 2024-11-13T18:38:53,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:53,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:53,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-13T18:38:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-13T18:38:53,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-13T18:38:53,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5580 sec 2024-11-13T18:38:53,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.5730 sec 2024-11-13T18:38:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:53,935 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:53,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T18:38:53,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a680374169b6495c8e9f795c33328010_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523133935/Put/seqid=0 2024-11-13T18:38:53,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742374_1550 (size=14594) 2024-11-13T18:38:53,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523193966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523193970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523193972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523193973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523194084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523194084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523194084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523194084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,102 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:38:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T18:38:54,264 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:54,264 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-13T18:38:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:38:54,266 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:54,266 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:54,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:54,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523194293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523194293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523194293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523194293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523194313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,317 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_1' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,354 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:54,357 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a680374169b6495c8e9f795c33328010_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a680374169b6495c8e9f795c33328010_21be930938e864d73f0be2877df0a945 2024-11-13T18:38:54,358 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/1f1686b6a9084dd3bd3de0c803db68cb, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:54,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/1f1686b6a9084dd3bd3de0c803db68cb is 175, key is test_row_0/A:col10/1731523133935/Put/seqid=0 2024-11-13T18:38:54,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742375_1551 (size=39549) 2024-11-13T18:38:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:38:54,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:38:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:54,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:38:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:54,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:38:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523194603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523194603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523194603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523194604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:54,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:54,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:38:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:54,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,762 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/1f1686b6a9084dd3bd3de0c803db68cb 2024-11-13T18:38:54,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/2801dbe1521a42068b3afac4f2415836 is 50, key is test_row_0/B:col10/1731523133935/Put/seqid=0 2024-11-13T18:38:54,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742376_1552 (size=12001) 2024-11-13T18:38:54,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/2801dbe1521a42068b3afac4f2415836 2024-11-13T18:38:54,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/3b601b0e58d14c5d86a9c46fbc906a11 is 50, key is test_row_0/C:col10/1731523133935/Put/seqid=0 2024-11-13T18:38:54,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742377_1553 (size=12001) 2024-11-13T18:38:54,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:38:54,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:54,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:54,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:54,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:38:55,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:38:55,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:55,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:55,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:55,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:55,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523195113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523195113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:55,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:55,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523195114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523195114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:55,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:38:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:55,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:55,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/3b601b0e58d14c5d86a9c46fbc906a11 2024-11-13T18:38:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:55,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/1f1686b6a9084dd3bd3de0c803db68cb as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1f1686b6a9084dd3bd3de0c803db68cb 2024-11-13T18:38:55,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1f1686b6a9084dd3bd3de0c803db68cb, entries=200, sequenceid=54, filesize=38.6 K 2024-11-13T18:38:55,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/2801dbe1521a42068b3afac4f2415836 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/2801dbe1521a42068b3afac4f2415836 2024-11-13T18:38:55,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/2801dbe1521a42068b3afac4f2415836, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T18:38:55,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/3b601b0e58d14c5d86a9c46fbc906a11 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3b601b0e58d14c5d86a9c46fbc906a11 2024-11-13T18:38:55,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3b601b0e58d14c5d86a9c46fbc906a11, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T18:38:55,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 21be930938e864d73f0be2877df0a945 in 1273ms, sequenceid=54, compaction requested=true 2024-11-13T18:38:55,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:55,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:55,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:55,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:55,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:55,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:55,208 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:55,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:55,208 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:55,209 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:55,209 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:55,209 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:38:55,209 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:38:55,209 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,209 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,210 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3b435627f8cb4ab1b53bca26062c9558, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b7e4c9526ada43d88950db2d422b60b9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/2801dbe1521a42068b3afac4f2415836] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=35.2 K 2024-11-13T18:38:55,210 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f2fb258e664624a75a608af63417c9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/386486a82a2849b4bebc8b501d660262, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1f1686b6a9084dd3bd3de0c803db68cb] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=99.1 K 2024-11-13T18:38:55,210 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,210 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f2fb258e664624a75a608af63417c9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/386486a82a2849b4bebc8b501d660262, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1f1686b6a9084dd3bd3de0c803db68cb] 2024-11-13T18:38:55,210 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b435627f8cb4ab1b53bca26062c9558, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731523132115 2024-11-13T18:38:55,210 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f2fb258e664624a75a608af63417c9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731523132115 2024-11-13T18:38:55,211 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e4c9526ada43d88950db2d422b60b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731523132144 2024-11-13T18:38:55,211 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 386486a82a2849b4bebc8b501d660262, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731523132144 2024-11-13T18:38:55,211 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2801dbe1521a42068b3afac4f2415836, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523133302 2024-11-13T18:38:55,211 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1686b6a9084dd3bd3de0c803db68cb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523133302 2024-11-13T18:38:55,216 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:55,218 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113256f0950f59c46fbb2f2c512d8f2a14b_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:55,220 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:55,220 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/7ce6b44543374a9192399207e99d5db2 is 50, key is test_row_0/B:col10/1731523133935/Put/seqid=0 2024-11-13T18:38:55,220 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113256f0950f59c46fbb2f2c512d8f2a14b_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:55,220 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113256f0950f59c46fbb2f2c512d8f2a14b_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:55,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742378_1554 (size=12104) 2024-11-13T18:38:55,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742379_1555 (size=4469) 2024-11-13T18:38:55,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T18:38:55,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,338 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T18:38:55,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134648b51cb6fb4a718e82f459c9fbd554_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523133970/Put/seqid=0 2024-11-13T18:38:55,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742380_1556 (size=12154) 2024-11-13T18:38:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:38:55,637 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#473 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:55,637 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/bf737576d82949d7be5e24a499fab584 is 175, key is test_row_0/A:col10/1731523133935/Put/seqid=0 2024-11-13T18:38:55,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742381_1557 (size=31058) 2024-11-13T18:38:55,649 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/7ce6b44543374a9192399207e99d5db2 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/7ce6b44543374a9192399207e99d5db2 2024-11-13T18:38:55,653 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into 7ce6b44543374a9192399207e99d5db2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:55,654 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:55,654 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523135208; duration=0sec 2024-11-13T18:38:55,654 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:55,654 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:38:55,654 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:55,655 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:55,655 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:38:55,655 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:55,655 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/2e216adf21d3470a89f4bea3e78e5319, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e59d7c333b7348cb947360efb41450d6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3b601b0e58d14c5d86a9c46fbc906a11] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=35.2 K 2024-11-13T18:38:55,655 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e216adf21d3470a89f4bea3e78e5319, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731523132115 2024-11-13T18:38:55,656 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e59d7c333b7348cb947360efb41450d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731523132144 2024-11-13T18:38:55,656 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b601b0e58d14c5d86a9c46fbc906a11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523133302 2024-11-13T18:38:55,663 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:55,664 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/56ed842c5cc34dc9a91244b45a156ed1 is 50, key is test_row_0/C:col10/1731523133935/Put/seqid=0 2024-11-13T18:38:55,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742382_1558 (size=12104) 2024-11-13T18:38:55,680 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/56ed842c5cc34dc9a91244b45a156ed1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/56ed842c5cc34dc9a91244b45a156ed1 2024-11-13T18:38:55,685 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into 56ed842c5cc34dc9a91244b45a156ed1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:55,685 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:55,685 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523135208; duration=0sec 2024-11-13T18:38:55,685 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:55,685 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:38:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:55,754 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134648b51cb6fb4a718e82f459c9fbd554_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134648b51cb6fb4a718e82f459c9fbd554_21be930938e864d73f0be2877df0a945 2024-11-13T18:38:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/53ff9078c54a4e4580a72fe8f7a97b8a, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:55,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/53ff9078c54a4e4580a72fe8f7a97b8a is 175, key is test_row_0/A:col10/1731523133970/Put/seqid=0 2024-11-13T18:38:55,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742383_1559 (size=30955) 2024-11-13T18:38:55,771 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/53ff9078c54a4e4580a72fe8f7a97b8a 2024-11-13T18:38:55,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/bbef9e4f4592440d83bc6aec2cf8e7f3 is 50, key is test_row_0/B:col10/1731523133970/Put/seqid=0 2024-11-13T18:38:55,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742384_1560 (size=12001) 2024-11-13T18:38:55,795 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/bbef9e4f4592440d83bc6aec2cf8e7f3 2024-11-13T18:38:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/ce915e4cdd5a410593ea01283b0ae663 is 50, key is test_row_0/C:col10/1731523133970/Put/seqid=0 2024-11-13T18:38:55,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742385_1561 (size=12001) 2024-11-13T18:38:56,054 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/bf737576d82949d7be5e24a499fab584 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bf737576d82949d7be5e24a499fab584 2024-11-13T18:38:56,058 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into bf737576d82949d7be5e24a499fab584(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:56,058 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:56,058 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523135208; duration=0sec 2024-11-13T18:38:56,058 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:56,058 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:38:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:56,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:56,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523196136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523196138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523196140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523196140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,206 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/ce915e4cdd5a410593ea01283b0ae663 2024-11-13T18:38:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/53ff9078c54a4e4580a72fe8f7a97b8a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53ff9078c54a4e4580a72fe8f7a97b8a 2024-11-13T18:38:56,216 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53ff9078c54a4e4580a72fe8f7a97b8a, entries=150, sequenceid=78, filesize=30.2 K 2024-11-13T18:38:56,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/bbef9e4f4592440d83bc6aec2cf8e7f3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bbef9e4f4592440d83bc6aec2cf8e7f3 2024-11-13T18:38:56,220 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bbef9e4f4592440d83bc6aec2cf8e7f3, entries=150, sequenceid=78, filesize=11.7 K 2024-11-13T18:38:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/ce915e4cdd5a410593ea01283b0ae663 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ce915e4cdd5a410593ea01283b0ae663 2024-11-13T18:38:56,232 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ce915e4cdd5a410593ea01283b0ae663, entries=150, sequenceid=78, filesize=11.7 K 2024-11-13T18:38:56,232 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 21be930938e864d73f0be2877df0a945 in 894ms, sequenceid=78, compaction requested=false 2024-11-13T18:38:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-13T18:38:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-13T18:38:56,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-13T18:38:56,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9670 sec 2024-11-13T18:38:56,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.9720 sec 2024-11-13T18:38:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:56,256 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:38:56,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T18:38:56,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111383515a60190c4968966350c1aa797d63_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523136255/Put/seqid=0 2024-11-13T18:38:56,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742386_1562 (size=17034) 2024-11-13T18:38:56,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523196278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523196280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523196281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523196282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523196333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,338 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_1' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:38:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523196393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523196393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523196393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523196393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T18:38:56,414 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:38:56,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:38:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-13T18:38:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:38:56,416 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:38:56,417 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:38:56,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:38:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:38:56,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:56,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523196603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523196603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523196603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523196603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,673 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:56,676 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111383515a60190c4968966350c1aa797d63_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383515a60190c4968966350c1aa797d63_21be930938e864d73f0be2877df0a945 2024-11-13T18:38:56,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/53e5a3eaabe64376900dc945f17af8af, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:56,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/53e5a3eaabe64376900dc945f17af8af is 175, key is test_row_0/A:col10/1731523136255/Put/seqid=0 2024-11-13T18:38:56,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742387_1563 (size=48139) 2024-11-13T18:38:56,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:56,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:38:56,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:56,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:56,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523196913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523196913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523196914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523196914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:57,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:57,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:57,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:38:57,081 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/53e5a3eaabe64376900dc945f17af8af 2024-11-13T18:38:57,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/cd4951b6a0b4404392cce78d2301d7bd is 50, key is test_row_0/B:col10/1731523136255/Put/seqid=0 2024-11-13T18:38:57,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742388_1564 (size=12001) 2024-11-13T18:38:57,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523197425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523197425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523197425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:57,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523197426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:57,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:57,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:57,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/cd4951b6a0b4404392cce78d2301d7bd 2024-11-13T18:38:57,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/b4228241de924354bb481a3146a07e98 is 50, key is test_row_0/C:col10/1731523136255/Put/seqid=0 2024-11-13T18:38:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742389_1565 (size=12001) 2024-11-13T18:38:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:38:57,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:57,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:57,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:38:57,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/b4228241de924354bb481a3146a07e98 2024-11-13T18:38:57,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/53e5a3eaabe64376900dc945f17af8af as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53e5a3eaabe64376900dc945f17af8af 2024-11-13T18:38:57,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53e5a3eaabe64376900dc945f17af8af, entries=250, sequenceid=98, filesize=47.0 K 2024-11-13T18:38:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/cd4951b6a0b4404392cce78d2301d7bd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/cd4951b6a0b4404392cce78d2301d7bd 2024-11-13T18:38:57,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/cd4951b6a0b4404392cce78d2301d7bd, entries=150, sequenceid=98, filesize=11.7 K 2024-11-13T18:38:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/b4228241de924354bb481a3146a07e98 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/b4228241de924354bb481a3146a07e98 2024-11-13T18:38:57,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/b4228241de924354bb481a3146a07e98, entries=150, sequenceid=98, filesize=11.7 K 2024-11-13T18:38:57,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 21be930938e864d73f0be2877df0a945 in 1661ms, sequenceid=98, compaction requested=true 2024-11-13T18:38:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:38:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:38:57,917 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:38:57,917 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:57,918 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:57,919 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:38:57,919 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,919 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/7ce6b44543374a9192399207e99d5db2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bbef9e4f4592440d83bc6aec2cf8e7f3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/cd4951b6a0b4404392cce78d2301d7bd] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=35.3 K 2024-11-13T18:38:57,919 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:57,919 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:38:57,919 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,919 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bf737576d82949d7be5e24a499fab584, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53ff9078c54a4e4580a72fe8f7a97b8a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53e5a3eaabe64376900dc945f17af8af] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=107.6 K 2024-11-13T18:38:57,919 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce6b44543374a9192399207e99d5db2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523133302 2024-11-13T18:38:57,919 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,919 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bf737576d82949d7be5e24a499fab584, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53ff9078c54a4e4580a72fe8f7a97b8a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53e5a3eaabe64376900dc945f17af8af] 2024-11-13T18:38:57,919 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting bbef9e4f4592440d83bc6aec2cf8e7f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523133969 2024-11-13T18:38:57,919 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf737576d82949d7be5e24a499fab584, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523133302 2024-11-13T18:38:57,920 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4951b6a0b4404392cce78d2301d7bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731523136138 2024-11-13T18:38:57,920 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53ff9078c54a4e4580a72fe8f7a97b8a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523133969 2024-11-13T18:38:57,920 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53e5a3eaabe64376900dc945f17af8af, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731523136138 2024-11-13T18:38:57,925 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:57,926 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:57,927 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113465c35cfca944f328f0ac11421d3bfe9_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:57,927 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/984916fc634946daa8a88b2e228f205b is 50, key is test_row_0/B:col10/1731523136255/Put/seqid=0 2024-11-13T18:38:57,928 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113465c35cfca944f328f0ac11421d3bfe9_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:57,928 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113465c35cfca944f328f0ac11421d3bfe9_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:57,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T18:38:57,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,945 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T18:38:57,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742390_1566 (size=12207) 2024-11-13T18:38:57,957 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/984916fc634946daa8a88b2e228f205b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/984916fc634946daa8a88b2e228f205b 2024-11-13T18:38:57,961 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into 984916fc634946daa8a88b2e228f205b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:57,961 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:57,961 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523137917; duration=0sec 2024-11-13T18:38:57,961 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:38:57,961 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:38:57,961 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:38:57,962 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:38:57,962 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:38:57,962 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:57,962 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/56ed842c5cc34dc9a91244b45a156ed1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ce915e4cdd5a410593ea01283b0ae663, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/b4228241de924354bb481a3146a07e98] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=35.3 K 2024-11-13T18:38:57,963 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ed842c5cc34dc9a91244b45a156ed1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731523133302 2024-11-13T18:38:57,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742391_1567 (size=4469) 2024-11-13T18:38:57,963 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ce915e4cdd5a410593ea01283b0ae663, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731523133969 2024-11-13T18:38:57,963 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b4228241de924354bb481a3146a07e98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731523136138 2024-11-13T18:38:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111321caf8b1022e4a6cb11d0f567251d594_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523136270/Put/seqid=0 2024-11-13T18:38:57,972 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:57,973 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e0e692f36673430abece9d8d8391dc0e is 50, key is test_row_0/C:col10/1731523136255/Put/seqid=0 2024-11-13T18:38:57,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742392_1568 (size=12154) 2024-11-13T18:38:57,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742393_1569 (size=12207) 2024-11-13T18:38:58,364 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#482 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:38:58,365 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/fe5dfc94d9dc4fba943be238dbdfef5e is 175, key is test_row_0/A:col10/1731523136255/Put/seqid=0 2024-11-13T18:38:58,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742394_1570 (size=31161) 2024-11-13T18:38:58,373 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/fe5dfc94d9dc4fba943be238dbdfef5e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fe5dfc94d9dc4fba943be238dbdfef5e 2024-11-13T18:38:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:38:58,388 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into fe5dfc94d9dc4fba943be238dbdfef5e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:58,388 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:58,388 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523137917; duration=0sec 2024-11-13T18:38:58,388 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:58,388 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:38:58,389 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111321caf8b1022e4a6cb11d0f567251d594_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111321caf8b1022e4a6cb11d0f567251d594_21be930938e864d73f0be2877df0a945 2024-11-13T18:38:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/edf1bea2774146a69f923d9f3f21ab4c, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:38:58,390 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e0e692f36673430abece9d8d8391dc0e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0e692f36673430abece9d8d8391dc0e 2024-11-13T18:38:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/edf1bea2774146a69f923d9f3f21ab4c is 175, key is test_row_0/A:col10/1731523136270/Put/seqid=0 2024-11-13T18:38:58,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742395_1571 (size=30955) 2024-11-13T18:38:58,396 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into e0e692f36673430abece9d8d8391dc0e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:38:58,397 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:58,397 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523137917; duration=0sec 2024-11-13T18:38:58,397 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:38:58,397 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:38:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:38:58,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:38:58,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523198455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523198456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523198458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523198460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:38:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523198564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523198564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523198573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523198573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523198774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523198774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523198783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523198783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:58,796 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/edf1bea2774146a69f923d9f3f21ab4c 2024-11-13T18:38:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/a09389fab5f6480188898e802adfd56a is 50, key is test_row_0/B:col10/1731523136270/Put/seqid=0 2024-11-13T18:38:58,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742396_1572 (size=12001) 2024-11-13T18:38:59,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523199083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:59,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523199083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:59,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523199093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523199094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:59,206 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/a09389fab5f6480188898e802adfd56a 2024-11-13T18:38:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/52f6ed441bdd4c35a284140cc3b08c45 is 50, key is test_row_0/C:col10/1731523136270/Put/seqid=0 2024-11-13T18:38:59,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742397_1573 (size=12001) 2024-11-13T18:38:59,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523199593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:59,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523199593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:59,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523199604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:59,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:38:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523199604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:38:59,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:38:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:38:59,621 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/52f6ed441bdd4c35a284140cc3b08c45 2024-11-13T18:38:59,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/edf1bea2774146a69f923d9f3f21ab4c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/edf1bea2774146a69f923d9f3f21ab4c 2024-11-13T18:38:59,629 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/edf1bea2774146a69f923d9f3f21ab4c, entries=150, sequenceid=117, filesize=30.2 K 2024-11-13T18:38:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/a09389fab5f6480188898e802adfd56a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/a09389fab5f6480188898e802adfd56a 2024-11-13T18:38:59,633 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/a09389fab5f6480188898e802adfd56a, entries=150, sequenceid=117, filesize=11.7 K 2024-11-13T18:38:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/52f6ed441bdd4c35a284140cc3b08c45 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/52f6ed441bdd4c35a284140cc3b08c45 2024-11-13T18:38:59,638 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/52f6ed441bdd4c35a284140cc3b08c45, entries=150, sequenceid=117, filesize=11.7 K 2024-11-13T18:38:59,638 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 21be930938e864d73f0be2877df0a945 in 1693ms, sequenceid=117, compaction requested=false 2024-11-13T18:38:59,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:38:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:38:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-13T18:38:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-13T18:38:59,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-13T18:38:59,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2230 sec 2024-11-13T18:38:59,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.2280 sec 2024-11-13T18:39:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:00,366 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:39:00,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T18:39:00,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132d2710a7bab54c99b01fe26ee1b44b73_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523138457/Put/seqid=0 2024-11-13T18:39:00,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742398_1574 (size=12304) 2024-11-13T18:39:00,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523200443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:00,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523200553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T18:39:00,573 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:00,574 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-13T18:39:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:39:00,575 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:00,576 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:00,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:00,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523200603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:00,608 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_1' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:00,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523200613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:00,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523200613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:00,616 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,616 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523200623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:00,627 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_0' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:39:00,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:00,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:00,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:00,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:00,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:00,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:00,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523200763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:00,778 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:00,781 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132d2710a7bab54c99b01fe26ee1b44b73_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d2710a7bab54c99b01fe26ee1b44b73_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:00,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/c3069290613b411ca678115058a13b9b, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:00,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/c3069290613b411ca678115058a13b9b is 175, key is test_row_0/A:col10/1731523138457/Put/seqid=0 2024-11-13T18:39:00,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742399_1575 (size=31105) 2024-11-13T18:39:00,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:00,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:00,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:00,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:00,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:39:01,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:01,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:01,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523201074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:01,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/c3069290613b411ca678115058a13b9b 2024-11-13T18:39:01,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:01,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:01,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/d09b7e14fa0140fe986f4abc587760f5 is 50, key is test_row_0/B:col10/1731523138457/Put/seqid=0 2024-11-13T18:39:01,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742400_1576 (size=12151) 2024-11-13T18:39:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:39:01,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:01,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:01,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523201594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:01,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/d09b7e14fa0140fe986f4abc587760f5 2024-11-13T18:39:01,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/1fe9697a87984642a2a8d39071c50fb5 is 50, key is test_row_0/C:col10/1731523138457/Put/seqid=0 2024-11-13T18:39:01,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742401_1577 (size=12151) 2024-11-13T18:39:01,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:01,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:01,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:39:01,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:01,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:01,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:01,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:02,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/1fe9697a87984642a2a8d39071c50fb5 2024-11-13T18:39:02,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/c3069290613b411ca678115058a13b9b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/c3069290613b411ca678115058a13b9b 2024-11-13T18:39:02,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/c3069290613b411ca678115058a13b9b, entries=150, sequenceid=138, filesize=30.4 K 2024-11-13T18:39:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/d09b7e14fa0140fe986f4abc587760f5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/d09b7e14fa0140fe986f4abc587760f5 2024-11-13T18:39:02,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/d09b7e14fa0140fe986f4abc587760f5, entries=150, sequenceid=138, filesize=11.9 K 2024-11-13T18:39:02,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/1fe9697a87984642a2a8d39071c50fb5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/1fe9697a87984642a2a8d39071c50fb5 2024-11-13T18:39:02,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/1fe9697a87984642a2a8d39071c50fb5, entries=150, sequenceid=138, filesize=11.9 K 2024-11-13T18:39:02,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 21be930938e864d73f0be2877df0a945 in 1666ms, sequenceid=138, compaction requested=true 2024-11-13T18:39:02,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:02,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:39:02,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:02,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:39:02,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:02,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:39:02,033 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:02,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:02,033 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:02,033 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:02,033 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:02,034 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:39:02,034 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:39:02,034 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:02,034 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:02,034 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/984916fc634946daa8a88b2e228f205b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/a09389fab5f6480188898e802adfd56a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/d09b7e14fa0140fe986f4abc587760f5] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=35.5 K 2024-11-13T18:39:02,034 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fe5dfc94d9dc4fba943be238dbdfef5e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/edf1bea2774146a69f923d9f3f21ab4c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/c3069290613b411ca678115058a13b9b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=91.0 K 2024-11-13T18:39:02,034 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:02,034 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fe5dfc94d9dc4fba943be238dbdfef5e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/edf1bea2774146a69f923d9f3f21ab4c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/c3069290613b411ca678115058a13b9b] 2024-11-13T18:39:02,034 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 984916fc634946daa8a88b2e228f205b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731523136138 2024-11-13T18:39:02,034 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a09389fab5f6480188898e802adfd56a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731523136270 2024-11-13T18:39:02,035 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting d09b7e14fa0140fe986f4abc587760f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523138454 2024-11-13T18:39:02,035 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe5dfc94d9dc4fba943be238dbdfef5e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731523136138 2024-11-13T18:39:02,037 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf1bea2774146a69f923d9f3f21ab4c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731523136270 2024-11-13T18:39:02,038 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3069290613b411ca678115058a13b9b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523138454 2024-11-13T18:39:02,047 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:02,048 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/76e8b866886c405db228dbc3190e146e is 50, key is test_row_0/B:col10/1731523138457/Put/seqid=0 2024-11-13T18:39:02,051 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:02,053 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135a5ca4b2c9b44208aa09a02352e4f40d_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:02,055 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135a5ca4b2c9b44208aa09a02352e4f40d_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:02,055 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135a5ca4b2c9b44208aa09a02352e4f40d_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:02,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742402_1578 (size=12459) 2024-11-13T18:39:02,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742403_1579 (size=4469) 2024-11-13T18:39:02,070 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#492 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:02,071 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/52995ec92c3b41a2ad95abeae507d060 is 175, key is test_row_0/A:col10/1731523138457/Put/seqid=0 2024-11-13T18:39:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742404_1580 (size=31413) 2024-11-13T18:39:02,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T18:39:02,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:02,107 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T18:39:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d9e86dea17564cc6be1e0b26691cde29_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523140412/Put/seqid=0 2024-11-13T18:39:02,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742405_1581 (size=12304) 2024-11-13T18:39:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,128 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d9e86dea17564cc6be1e0b26691cde29_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d9e86dea17564cc6be1e0b26691cde29_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:02,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/1a0b2672aaef49eb81f938ef40f83961, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:02,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/1a0b2672aaef49eb81f938ef40f83961 is 175, key is test_row_0/A:col10/1731523140412/Put/seqid=0 2024-11-13T18:39:02,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742406_1582 (size=31105) 2024-11-13T18:39:02,463 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/76e8b866886c405db228dbc3190e146e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/76e8b866886c405db228dbc3190e146e 2024-11-13T18:39:02,467 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into 76e8b866886c405db228dbc3190e146e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:02,468 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:02,468 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523142033; duration=0sec 2024-11-13T18:39:02,468 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:02,468 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:39:02,468 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:02,469 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:02,469 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:39:02,470 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:02,470 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0e692f36673430abece9d8d8391dc0e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/52f6ed441bdd4c35a284140cc3b08c45, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/1fe9697a87984642a2a8d39071c50fb5] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=35.5 K 2024-11-13T18:39:02,470 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e692f36673430abece9d8d8391dc0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731523136138 2024-11-13T18:39:02,470 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 52f6ed441bdd4c35a284140cc3b08c45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731523136270 2024-11-13T18:39:02,471 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe9697a87984642a2a8d39071c50fb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523138454 2024-11-13T18:39:02,477 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:02,477 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/446f18683de74b4db6554979aeb244c5 is 50, key is test_row_0/C:col10/1731523138457/Put/seqid=0 2024-11-13T18:39:02,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742407_1583 (size=12459) 2024-11-13T18:39:02,483 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/52995ec92c3b41a2ad95abeae507d060 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/52995ec92c3b41a2ad95abeae507d060 2024-11-13T18:39:02,487 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/446f18683de74b4db6554979aeb244c5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/446f18683de74b4db6554979aeb244c5 2024-11-13T18:39:02,488 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into 52995ec92c3b41a2ad95abeae507d060(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:02,488 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:02,488 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523142032; duration=0sec 2024-11-13T18:39:02,488 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:02,488 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:39:02,497 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into 446f18683de74b4db6554979aeb244c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:02,497 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:02,497 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523142033; duration=0sec 2024-11-13T18:39:02,497 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:02,497 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:39:02,533 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/1a0b2672aaef49eb81f938ef40f83961 2024-11-13T18:39:02,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/79b2dc1965ea43d0947b1d028e74e935 is 50, key is test_row_0/B:col10/1731523140412/Put/seqid=0 2024-11-13T18:39:02,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742408_1584 (size=12151) 2024-11-13T18:39:02,556 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/79b2dc1965ea43d0947b1d028e74e935 2024-11-13T18:39:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/bfb1550f6d674e9ebddeb43445779af1 is 50, key is test_row_0/C:col10/1731523140412/Put/seqid=0 2024-11-13T18:39:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742409_1585 (size=12151) 2024-11-13T18:39:02,577 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/bfb1550f6d674e9ebddeb43445779af1 2024-11-13T18:39:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/1a0b2672aaef49eb81f938ef40f83961 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1a0b2672aaef49eb81f938ef40f83961 2024-11-13T18:39:02,585 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1a0b2672aaef49eb81f938ef40f83961, entries=150, sequenceid=156, filesize=30.4 K 2024-11-13T18:39:02,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/79b2dc1965ea43d0947b1d028e74e935 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/79b2dc1965ea43d0947b1d028e74e935 2024-11-13T18:39:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,590 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/79b2dc1965ea43d0947b1d028e74e935, entries=150, sequenceid=156, filesize=11.9 K 2024-11-13T18:39:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/bfb1550f6d674e9ebddeb43445779af1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/bfb1550f6d674e9ebddeb43445779af1 2024-11-13T18:39:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,594 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/bfb1550f6d674e9ebddeb43445779af1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-13T18:39:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,595 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 21be930938e864d73f0be2877df0a945 in 488ms, sequenceid=156, compaction requested=false 2024-11-13T18:39:02,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:02,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:02,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-13T18:39:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-13T18:39:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-13T18:39:02,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0200 sec 2024-11-13T18:39:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0260 sec 2024-11-13T18:39:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:02,668 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:39:02,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:39:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:02,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a1bf7500e7624f51a646a521263f905f_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523142663/Put/seqid=0 2024-11-13T18:39:02,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742411_1587 (size=24758) 2024-11-13T18:39:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T18:39:02,723 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:02,724 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-13T18:39:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:39:02,725 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:02,726 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:02,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:02,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523202751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523202754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523202755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523202757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523202757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:39:02,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523202863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523202863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523202864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523202873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523202873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:02,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:39:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:02,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:39:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:39:03,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523203074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523203074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523203074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523203083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523203083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,090 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:03,094 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a1bf7500e7624f51a646a521263f905f_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a1bf7500e7624f51a646a521263f905f_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:03,094 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/bcfc0e2378c649b1ac741f49bb8381c6, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:03,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/bcfc0e2378c649b1ac741f49bb8381c6 is 175, key is test_row_0/A:col10/1731523142663/Put/seqid=0 2024-11-13T18:39:03,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742410_1586 (size=74395) 2024-11-13T18:39:03,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:39:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:39:03,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:03,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:39:03,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523203383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523203383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523203383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523203393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523203394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:39:03,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:03,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:03,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/bcfc0e2378c649b1ac741f49bb8381c6 2024-11-13T18:39:03,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c1c5314e2a0b480c95a25a0741a16f2a is 50, key is test_row_0/B:col10/1731523142663/Put/seqid=0 2024-11-13T18:39:03,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742412_1588 (size=12151) 2024-11-13T18:39:03,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c1c5314e2a0b480c95a25a0741a16f2a 2024-11-13T18:39:03,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/9b3c696f33a14f0bb41b7986ccb3b686 is 50, key is test_row_0/C:col10/1731523142663/Put/seqid=0 2024-11-13T18:39:03,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742413_1589 (size=12151) 2024-11-13T18:39:03,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/9b3c696f33a14f0bb41b7986ccb3b686 2024-11-13T18:39:03,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/bcfc0e2378c649b1ac741f49bb8381c6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bcfc0e2378c649b1ac741f49bb8381c6 2024-11-13T18:39:03,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bcfc0e2378c649b1ac741f49bb8381c6, entries=400, sequenceid=170, filesize=72.7 K 2024-11-13T18:39:03,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c1c5314e2a0b480c95a25a0741a16f2a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c1c5314e2a0b480c95a25a0741a16f2a 2024-11-13T18:39:03,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c1c5314e2a0b480c95a25a0741a16f2a, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T18:39:03,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/9b3c696f33a14f0bb41b7986ccb3b686 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/9b3c696f33a14f0bb41b7986ccb3b686 2024-11-13T18:39:03,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/9b3c696f33a14f0bb41b7986ccb3b686, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T18:39:03,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 21be930938e864d73f0be2877df0a945 in 875ms, sequenceid=170, compaction requested=true 2024-11-13T18:39:03,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:03,543 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:03,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:39:03,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:03,544 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:03,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:39:03,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:03,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:39:03,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:03,544 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136913 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:03,544 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:39:03,544 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,545 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/52995ec92c3b41a2ad95abeae507d060, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1a0b2672aaef49eb81f938ef40f83961, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bcfc0e2378c649b1ac741f49bb8381c6] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=133.7 K 2024-11-13T18:39:03,545 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,545 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/52995ec92c3b41a2ad95abeae507d060, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1a0b2672aaef49eb81f938ef40f83961, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bcfc0e2378c649b1ac741f49bb8381c6] 2024-11-13T18:39:03,545 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52995ec92c3b41a2ad95abeae507d060, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523138454 2024-11-13T18:39:03,545 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:03,545 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:39:03,545 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,545 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a0b2672aaef49eb81f938ef40f83961, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731523140412 2024-11-13T18:39:03,546 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/76e8b866886c405db228dbc3190e146e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/79b2dc1965ea43d0947b1d028e74e935, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c1c5314e2a0b480c95a25a0741a16f2a] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=35.9 K 2024-11-13T18:39:03,546 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcfc0e2378c649b1ac741f49bb8381c6, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523142624 2024-11-13T18:39:03,546 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 76e8b866886c405db228dbc3190e146e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523138454 2024-11-13T18:39:03,547 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b2dc1965ea43d0947b1d028e74e935, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731523140412 2024-11-13T18:39:03,548 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c5314e2a0b480c95a25a0741a16f2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523142655 2024-11-13T18:39:03,565 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:03,573 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#501 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:03,574 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/429d4314867944e3b951689408be9e81 is 50, key is test_row_0/B:col10/1731523142663/Put/seqid=0 2024-11-13T18:39:03,578 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cfd8641e41434689a6fd60427f9d24d7_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:03,580 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cfd8641e41434689a6fd60427f9d24d7_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:03,580 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cfd8641e41434689a6fd60427f9d24d7_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:03,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742414_1590 (size=12561) 2024-11-13T18:39:03,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742415_1591 (size=4469) 2024-11-13T18:39:03,596 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#500 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:03,597 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/5bee6beeffdb43af8a9b69b0cf53d645 is 175, key is test_row_0/A:col10/1731523142663/Put/seqid=0 2024-11-13T18:39:03,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742416_1592 (size=31515) 2024-11-13T18:39:03,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T18:39:03,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,646 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:39:03,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133f78dd80b55645329d25effee6a37b1a_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523142755/Put/seqid=0 2024-11-13T18:39:03,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742417_1593 (size=12304) 2024-11-13T18:39:03,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:03,678 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133f78dd80b55645329d25effee6a37b1a_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f78dd80b55645329d25effee6a37b1a_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/800005fa980142798a3ed54062cee20f, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:03,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/800005fa980142798a3ed54062cee20f is 175, key is test_row_0/A:col10/1731523142755/Put/seqid=0 2024-11-13T18:39:03,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742418_1594 (size=31105) 2024-11-13T18:39:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:39:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:03,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523203903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523203903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523203903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523203905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523203906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:03,991 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/429d4314867944e3b951689408be9e81 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/429d4314867944e3b951689408be9e81 2024-11-13T18:39:03,996 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into 429d4314867944e3b951689408be9e81(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:03,996 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:03,996 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523143544; duration=0sec 2024-11-13T18:39:03,996 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:03,996 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:39:03,996 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:03,997 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:03,997 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:39:03,997 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:03,997 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/446f18683de74b4db6554979aeb244c5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/bfb1550f6d674e9ebddeb43445779af1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/9b3c696f33a14f0bb41b7986ccb3b686] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=35.9 K 2024-11-13T18:39:03,998 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 446f18683de74b4db6554979aeb244c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731523138454 2024-11-13T18:39:03,999 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb1550f6d674e9ebddeb43445779af1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731523140412 2024-11-13T18:39:04,000 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b3c696f33a14f0bb41b7986ccb3b686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523142655 2024-11-13T18:39:04,007 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/5bee6beeffdb43af8a9b69b0cf53d645 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5bee6beeffdb43af8a9b69b0cf53d645 2024-11-13T18:39:04,009 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:04,010 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/375edf0bd3444d8dacac0d0c713dd5af is 50, key is test_row_0/C:col10/1731523142663/Put/seqid=0 2024-11-13T18:39:04,012 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into 5bee6beeffdb43af8a9b69b0cf53d645(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:04,012 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:04,012 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523143543; duration=0sec 2024-11-13T18:39:04,012 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:04,012 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:39:04,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523204013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523204013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742419_1595 (size=12561) 2024-11-13T18:39:04,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523204023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,087 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/800005fa980142798a3ed54062cee20f 2024-11-13T18:39:04,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/bd65d810bcd8459dbd8b1fdd2904ee0f is 50, key is test_row_0/B:col10/1731523142755/Put/seqid=0 2024-11-13T18:39:04,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742420_1596 (size=12151) 2024-11-13T18:39:04,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523204223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523204223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523204233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,428 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/375edf0bd3444d8dacac0d0c713dd5af as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/375edf0bd3444d8dacac0d0c713dd5af 2024-11-13T18:39:04,433 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into 375edf0bd3444d8dacac0d0c713dd5af(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:04,433 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:04,433 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523143544; duration=0sec 2024-11-13T18:39:04,433 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:04,433 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:39:04,498 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/bd65d810bcd8459dbd8b1fdd2904ee0f 2024-11-13T18:39:04,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/0135be243fed4852ad1b461e0c5944be is 50, key is test_row_0/C:col10/1731523142755/Put/seqid=0 2024-11-13T18:39:04,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742421_1597 (size=12151) 2024-11-13T18:39:04,513 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/0135be243fed4852ad1b461e0c5944be 2024-11-13T18:39:04,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/800005fa980142798a3ed54062cee20f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/800005fa980142798a3ed54062cee20f 2024-11-13T18:39:04,520 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/800005fa980142798a3ed54062cee20f, entries=150, sequenceid=196, filesize=30.4 K 2024-11-13T18:39:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/bd65d810bcd8459dbd8b1fdd2904ee0f as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bd65d810bcd8459dbd8b1fdd2904ee0f 2024-11-13T18:39:04,524 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bd65d810bcd8459dbd8b1fdd2904ee0f, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T18:39:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/0135be243fed4852ad1b461e0c5944be as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/0135be243fed4852ad1b461e0c5944be 2024-11-13T18:39:04,527 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/0135be243fed4852ad1b461e0c5944be, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T18:39:04,528 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 21be930938e864d73f0be2877df0a945 in 882ms, sequenceid=196, compaction requested=false 2024-11-13T18:39:04,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:04,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:04,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-13T18:39:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-13T18:39:04,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-13T18:39:04,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-13T18:39:04,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8080 sec 2024-11-13T18:39:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:04,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:39:04,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:39:04,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130adbf28658434d36956295da1e98ced7_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523143902/Put/seqid=0 2024-11-13T18:39:04,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742422_1598 (size=14794) 2024-11-13T18:39:04,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523204582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523204583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523204584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523204693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523204693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523204693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T18:39:04,874 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:04,874 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-13T18:39:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:39:04,876 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:04,877 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:04,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:04,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523204904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523204904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523204904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523204923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523204923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:04,926 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_1' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,925 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:04,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:04,949 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130adbf28658434d36956295da1e98ced7_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130adbf28658434d36956295da1e98ced7_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:04,950 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/79f8c7d5542b4c2e8cf0140b8753dcf8, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:04,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/79f8c7d5542b4c2e8cf0140b8753dcf8 is 175, key is test_row_0/A:col10/1731523143902/Put/seqid=0 2024-11-13T18:39:04,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742423_1599 (size=39749) 2024-11-13T18:39:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:39:05,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T18:39:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T18:39:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:39:05,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523205213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523205213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523205213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:05,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T18:39:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,354 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/79f8c7d5542b4c2e8cf0140b8753dcf8 2024-11-13T18:39:05,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/717a3310294641acaba928ae64a54b72 is 50, key is test_row_0/B:col10/1731523143902/Put/seqid=0 2024-11-13T18:39:05,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742424_1600 (size=12151) 2024-11-13T18:39:05,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/717a3310294641acaba928ae64a54b72 2024-11-13T18:39:05,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/12eac78505ad4758b848e58287dcfed1 is 50, key is test_row_0/C:col10/1731523143902/Put/seqid=0 2024-11-13T18:39:05,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742425_1601 (size=12151) 2024-11-13T18:39:05,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T18:39:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:39:05,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T18:39:05,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:05,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523205723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523205723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:05,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523205733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:05,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/12eac78505ad4758b848e58287dcfed1 2024-11-13T18:39:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/79f8c7d5542b4c2e8cf0140b8753dcf8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f8c7d5542b4c2e8cf0140b8753dcf8 2024-11-13T18:39:05,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f8c7d5542b4c2e8cf0140b8753dcf8, entries=200, sequenceid=210, filesize=38.8 K 2024-11-13T18:39:05,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/717a3310294641acaba928ae64a54b72 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/717a3310294641acaba928ae64a54b72 2024-11-13T18:39:05,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/717a3310294641acaba928ae64a54b72, entries=150, sequenceid=210, filesize=11.9 K 2024-11-13T18:39:05,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/12eac78505ad4758b848e58287dcfed1 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/12eac78505ad4758b848e58287dcfed1 2024-11-13T18:39:05,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/12eac78505ad4758b848e58287dcfed1, entries=150, sequenceid=210, filesize=11.9 K 2024-11-13T18:39:05,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 21be930938e864d73f0be2877df0a945 in 1256ms, sequenceid=210, compaction requested=true 2024-11-13T18:39:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:39:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:39:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:39:05,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:05,791 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:05,791 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:05,792 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:05,792 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:39:05,792 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,792 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/429d4314867944e3b951689408be9e81, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bd65d810bcd8459dbd8b1fdd2904ee0f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/717a3310294641acaba928ae64a54b72] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=36.0 K 2024-11-13T18:39:05,792 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:05,792 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:39:05,792 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,793 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5bee6beeffdb43af8a9b69b0cf53d645, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/800005fa980142798a3ed54062cee20f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f8c7d5542b4c2e8cf0140b8753dcf8] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=100.0 K 2024-11-13T18:39:05,793 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,793 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5bee6beeffdb43af8a9b69b0cf53d645, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/800005fa980142798a3ed54062cee20f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f8c7d5542b4c2e8cf0140b8753dcf8] 2024-11-13T18:39:05,793 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 429d4314867944e3b951689408be9e81, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523142655 2024-11-13T18:39:05,793 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bee6beeffdb43af8a9b69b0cf53d645, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523142655 2024-11-13T18:39:05,793 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting bd65d810bcd8459dbd8b1fdd2904ee0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731523142743 2024-11-13T18:39:05,794 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 800005fa980142798a3ed54062cee20f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731523142743 2024-11-13T18:39:05,794 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 717a3310294641acaba928ae64a54b72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523143902 2024-11-13T18:39:05,794 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f8c7d5542b4c2e8cf0140b8753dcf8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523143902 2024-11-13T18:39:05,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T18:39:05,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,798 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T18:39:05,804 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:05,804 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/dac2e0a33c87408d8b092e846a9f9290 is 50, key is test_row_0/B:col10/1731523143902/Put/seqid=0 2024-11-13T18:39:05,806 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:05,822 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113072b2fee4c754de48f8d03298e54fdc3_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:05,824 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113072b2fee4c754de48f8d03298e54fdc3_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:05,824 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113072b2fee4c754de48f8d03298e54fdc3_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:05,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742426_1602 (size=12663) 2024-11-13T18:39:05,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1a0e5b3bce646a59f9e9c53c38de578_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523144582/Put/seqid=0 2024-11-13T18:39:05,830 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/dac2e0a33c87408d8b092e846a9f9290 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/dac2e0a33c87408d8b092e846a9f9290 2024-11-13T18:39:05,837 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into dac2e0a33c87408d8b092e846a9f9290(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:05,837 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:05,837 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523145791; duration=0sec 2024-11-13T18:39:05,837 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:05,837 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:39:05,837 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:05,840 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:05,840 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:39:05,840 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,840 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/375edf0bd3444d8dacac0d0c713dd5af, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/0135be243fed4852ad1b461e0c5944be, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/12eac78505ad4758b848e58287dcfed1] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=36.0 K 2024-11-13T18:39:05,840 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 375edf0bd3444d8dacac0d0c713dd5af, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731523142655 2024-11-13T18:39:05,841 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0135be243fed4852ad1b461e0c5944be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731523142743 2024-11-13T18:39:05,841 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 12eac78505ad4758b848e58287dcfed1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523143902 2024-11-13T18:39:05,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742427_1603 (size=4469) 2024-11-13T18:39:05,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742428_1604 (size=12304) 2024-11-13T18:39:05,860 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:05,860 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/74bbed2c03f7422598932c4860b43d77 is 50, key is test_row_0/C:col10/1731523143902/Put/seqid=0 2024-11-13T18:39:05,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,865 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1a0e5b3bce646a59f9e9c53c38de578_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1a0e5b3bce646a59f9e9c53c38de578_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:05,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/fdb4284ce5d74d4ba260b69521003c8e, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:05,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/fdb4284ce5d74d4ba260b69521003c8e is 175, key is test_row_0/A:col10/1731523144582/Put/seqid=0 2024-11-13T18:39:05,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742429_1605 (size=12663) 2024-11-13T18:39:05,873 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/74bbed2c03f7422598932c4860b43d77 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/74bbed2c03f7422598932c4860b43d77 2024-11-13T18:39:05,879 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into 74bbed2c03f7422598932c4860b43d77(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:05,879 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:05,879 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523145791; duration=0sec 2024-11-13T18:39:05,879 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:05,879 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:39:05,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742430_1606 (size=31105) 2024-11-13T18:39:05,894 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/fdb4284ce5d74d4ba260b69521003c8e 2024-11-13T18:39:05,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/31f271454d0941e78f729667c62fafcb is 50, key is test_row_0/B:col10/1731523144582/Put/seqid=0 2024-11-13T18:39:05,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742431_1607 (size=12151) 2024-11-13T18:39:05,907 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/31f271454d0941e78f729667c62fafcb 2024-11-13T18:39:05,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/cf51c6f8cf4c4539b555f6648c4c26b5 is 50, key is test_row_0/C:col10/1731523144582/Put/seqid=0 2024-11-13T18:39:05,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742432_1608 (size=12151) 2024-11-13T18:39:05,925 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/cf51c6f8cf4c4539b555f6648c4c26b5 2024-11-13T18:39:05,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/fdb4284ce5d74d4ba260b69521003c8e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fdb4284ce5d74d4ba260b69521003c8e 2024-11-13T18:39:05,934 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fdb4284ce5d74d4ba260b69521003c8e, entries=150, sequenceid=236, filesize=30.4 K 2024-11-13T18:39:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/31f271454d0941e78f729667c62fafcb as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/31f271454d0941e78f729667c62fafcb 2024-11-13T18:39:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,939 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/31f271454d0941e78f729667c62fafcb, entries=150, sequenceid=236, filesize=11.9 K 2024-11-13T18:39:05,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/cf51c6f8cf4c4539b555f6648c4c26b5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/cf51c6f8cf4c4539b555f6648c4c26b5 2024-11-13T18:39:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,946 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/cf51c6f8cf4c4539b555f6648c4c26b5, entries=150, sequenceid=236, filesize=11.9 K 2024-11-13T18:39:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,948 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 21be930938e864d73f0be2877df0a945 in 150ms, sequenceid=236, compaction requested=false 2024-11-13T18:39:05,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:05,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:05,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-13T18:39:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-13T18:39:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-13T18:39:05,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-11-13T18:39:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0800 sec 2024-11-13T18:39:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T18:39:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,013 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-13T18:39:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,015 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:39:06,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:39:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T18:39:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-13T18:39:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-13T18:39:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-13T18:39:06,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-13T18:39:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 160 msec 2024-11-13T18:39:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,252 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#510 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,253 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/6a31d28054ce495f9030a60a7dbfc4ba is 175, key is test_row_0/A:col10/1731523143902/Put/seqid=0 2024-11-13T18:39:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742433_1609 (size=31617) 2024-11-13T18:39:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,268 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/6a31d28054ce495f9030a60a7dbfc4ba as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/6a31d28054ce495f9030a60a7dbfc4ba 2024-11-13T18:39:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,276 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into 6a31d28054ce495f9030a60a7dbfc4ba(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,276 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:06,276 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523145791; duration=0sec 2024-11-13T18:39:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,276 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:06,276 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:39:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T18:39:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,334 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,334 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-13T18:39:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,337 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:39:06,341 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:06,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:39:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T18:39:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-13T18:39:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-13T18:39:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-13T18:39:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-13T18:39:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 167 msec 2024-11-13T18:39:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T18:39:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,654 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,654 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-13T18:39:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,657 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,658 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:06,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:39:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:39:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:39:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T18:39:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ffaf53e3d61d409a8ca95f246ea53895_21be930938e864d73f0be2877df0a945 is 50, key is test_row_1/A:col10/1731523146793/Put/seqid=0 2024-11-13T18:39:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T18:39:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:06,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742434_1610 (size=22268) 2024-11-13T18:39:06,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:06,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ffaf53e3d61d409a8ca95f246ea53895_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ffaf53e3d61d409a8ca95f246ea53895_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:06,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/d10a3b97e83a4ab0b91ef682f60a61f5, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:06,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/d10a3b97e83a4ab0b91ef682f60a61f5 is 175, key is test_row_1/A:col10/1731523146793/Put/seqid=0 2024-11-13T18:39:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523206852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523206851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:06,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523206854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:06,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742435_1611 (size=65669) 2024-11-13T18:39:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523206943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:06,947 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_1' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523206953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:06,955 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:06,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523206964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:06,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523206964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:06,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T18:39:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:06,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523206964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:06,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:39:07,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T18:39:07,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:07,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523207173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:07,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523207174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:07,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523207174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:07,270 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/d10a3b97e83a4ab0b91ef682f60a61f5 2024-11-13T18:39:07,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T18:39:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c191edc170574ea58c48c51fafbd1355 is 50, key is test_row_1/B:col10/1731523146793/Put/seqid=0 2024-11-13T18:39:07,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742436_1612 (size=9757) 2024-11-13T18:39:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:39:07,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T18:39:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523207483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523207483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523207483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:07,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T18:39:07,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:07,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:07,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c191edc170574ea58c48c51fafbd1355 2024-11-13T18:39:07,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f6c53a4269e9441ab3a2d2539349a719 is 50, key is test_row_1/C:col10/1731523146793/Put/seqid=0 2024-11-13T18:39:07,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742437_1613 (size=9757) 2024-11-13T18:39:07,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f6c53a4269e9441ab3a2d2539349a719 2024-11-13T18:39:07,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/d10a3b97e83a4ab0b91ef682f60a61f5 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d10a3b97e83a4ab0b91ef682f60a61f5 2024-11-13T18:39:07,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d10a3b97e83a4ab0b91ef682f60a61f5, entries=350, sequenceid=252, filesize=64.1 K 2024-11-13T18:39:07,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c191edc170574ea58c48c51fafbd1355 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c191edc170574ea58c48c51fafbd1355 2024-11-13T18:39:07,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c191edc170574ea58c48c51fafbd1355, entries=100, sequenceid=252, filesize=9.5 K 2024-11-13T18:39:07,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f6c53a4269e9441ab3a2d2539349a719 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f6c53a4269e9441ab3a2d2539349a719 2024-11-13T18:39:07,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f6c53a4269e9441ab3a2d2539349a719, entries=100, sequenceid=252, filesize=9.5 K 2024-11-13T18:39:07,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 21be930938e864d73f0be2877df0a945 in 918ms, sequenceid=252, compaction requested=true 2024-11-13T18:39:07,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:07,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:39:07,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:07,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:39:07,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:07,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:39:07,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:07,712 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:07,712 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:07,713 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:07,713 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:07,713 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:39:07,713 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:39:07,713 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,713 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,713 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/6a31d28054ce495f9030a60a7dbfc4ba, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fdb4284ce5d74d4ba260b69521003c8e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d10a3b97e83a4ab0b91ef682f60a61f5] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=125.4 K 2024-11-13T18:39:07,713 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,713 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/dac2e0a33c87408d8b092e846a9f9290, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/31f271454d0941e78f729667c62fafcb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c191edc170574ea58c48c51fafbd1355] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=33.8 K 2024-11-13T18:39:07,713 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/6a31d28054ce495f9030a60a7dbfc4ba, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fdb4284ce5d74d4ba260b69521003c8e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d10a3b97e83a4ab0b91ef682f60a61f5] 2024-11-13T18:39:07,714 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting dac2e0a33c87408d8b092e846a9f9290, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523143902 2024-11-13T18:39:07,714 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a31d28054ce495f9030a60a7dbfc4ba, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523143902 2024-11-13T18:39:07,714 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdb4284ce5d74d4ba260b69521003c8e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731523144576 2024-11-13T18:39:07,714 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f271454d0941e78f729667c62fafcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731523144576 2024-11-13T18:39:07,715 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10a3b97e83a4ab0b91ef682f60a61f5, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731523146773 2024-11-13T18:39:07,715 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c191edc170574ea58c48c51fafbd1355, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731523146776 2024-11-13T18:39:07,719 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:07,720 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#518 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:07,720 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/04a01df0fe7e44ea9ebef965d66485a8 is 50, key is test_row_0/B:col10/1731523144582/Put/seqid=0 2024-11-13T18:39:07,726 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132800680019474dc2aa51573db3558eeb_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:07,727 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132800680019474dc2aa51573db3558eeb_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:07,727 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132800680019474dc2aa51573db3558eeb_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:07,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742438_1614 (size=12765) 2024-11-13T18:39:07,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T18:39:07,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,735 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T18:39:07,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742439_1615 (size=4469) 2024-11-13T18:39:07,740 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/04a01df0fe7e44ea9ebef965d66485a8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/04a01df0fe7e44ea9ebef965d66485a8 2024-11-13T18:39:07,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113498548f535104ac79a72051d771fe4f5_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523146852/Put/seqid=0 2024-11-13T18:39:07,746 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into 04a01df0fe7e44ea9ebef965d66485a8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:07,746 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:07,746 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523147712; duration=0sec 2024-11-13T18:39:07,746 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:07,746 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:39:07,746 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:07,747 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:07,747 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:39:07,747 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:07,747 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/74bbed2c03f7422598932c4860b43d77, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/cf51c6f8cf4c4539b555f6648c4c26b5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f6c53a4269e9441ab3a2d2539349a719] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=33.8 K 2024-11-13T18:39:07,747 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 74bbed2c03f7422598932c4860b43d77, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731523143902 2024-11-13T18:39:07,749 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting cf51c6f8cf4c4539b555f6648c4c26b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731523144576 2024-11-13T18:39:07,750 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting f6c53a4269e9441ab3a2d2539349a719, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731523146776 2024-11-13T18:39:07,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742440_1616 (size=12454) 2024-11-13T18:39:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:07,756 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113498548f535104ac79a72051d771fe4f5_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113498548f535104ac79a72051d771fe4f5_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:07,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/17d4ace58e694da9a91f602291a18c7b, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:07,758 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:07,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/17d4ace58e694da9a91f602291a18c7b is 175, key is test_row_0/A:col10/1731523146852/Put/seqid=0 2024-11-13T18:39:07,759 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e4d2924fe2e8411caba130c6fc894282 is 50, key is test_row_0/C:col10/1731523144582/Put/seqid=0 2024-11-13T18:39:07,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742441_1617 (size=31255) 2024-11-13T18:39:07,764 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/17d4ace58e694da9a91f602291a18c7b 2024-11-13T18:39:07,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742442_1618 (size=12765) 2024-11-13T18:39:07,769 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e4d2924fe2e8411caba130c6fc894282 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e4d2924fe2e8411caba130c6fc894282 2024-11-13T18:39:07,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/b07eb46ba79942b993cb793e86c09574 is 50, key is test_row_0/B:col10/1731523146852/Put/seqid=0 2024-11-13T18:39:07,773 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into e4d2924fe2e8411caba130c6fc894282(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:07,773 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:07,773 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523147712; duration=0sec 2024-11-13T18:39:07,773 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:07,773 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:39:07,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742443_1619 (size=12301) 2024-11-13T18:39:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:39:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:07,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:08,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523208006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523208007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523208007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523208113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523208123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523208123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,141 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#519 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:08,142 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/3388f145ca074a4793b5730cdaad7dc8 is 175, key is test_row_0/A:col10/1731523144582/Put/seqid=0 2024-11-13T18:39:08,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742444_1620 (size=31826) 2024-11-13T18:39:08,175 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/b07eb46ba79942b993cb793e86c09574 2024-11-13T18:39:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/a1f49625aa4e42f49a1389857628572e is 50, key is test_row_0/C:col10/1731523146852/Put/seqid=0 2024-11-13T18:39:08,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742445_1621 (size=12301) 2024-11-13T18:39:08,193 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/a1f49625aa4e42f49a1389857628572e 2024-11-13T18:39:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/17d4ace58e694da9a91f602291a18c7b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/17d4ace58e694da9a91f602291a18c7b 2024-11-13T18:39:08,201 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/17d4ace58e694da9a91f602291a18c7b, entries=150, sequenceid=277, filesize=30.5 K 2024-11-13T18:39:08,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/b07eb46ba79942b993cb793e86c09574 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b07eb46ba79942b993cb793e86c09574 2024-11-13T18:39:08,205 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b07eb46ba79942b993cb793e86c09574, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T18:39:08,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/a1f49625aa4e42f49a1389857628572e as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/a1f49625aa4e42f49a1389857628572e 2024-11-13T18:39:08,209 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/a1f49625aa4e42f49a1389857628572e, entries=150, sequenceid=277, filesize=12.0 K 2024-11-13T18:39:08,210 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 21be930938e864d73f0be2877df0a945 in 476ms, sequenceid=277, compaction requested=false 2024-11-13T18:39:08,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:08,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:08,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-13T18:39:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-13T18:39:08,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-13T18:39:08,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5530 sec 2024-11-13T18:39:08,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.5600 sec 2024-11-13T18:39:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:39:08,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T18:39:08,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f237354b4df64a9281828dbb0f65520a_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523148326/Put/seqid=0 2024-11-13T18:39:08,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742446_1622 (size=14994) 2024-11-13T18:39:08,342 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:08,346 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f237354b4df64a9281828dbb0f65520a_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f237354b4df64a9281828dbb0f65520a_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:08,347 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/203d5d0215194666bf1be7871efca3c0, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:08,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/203d5d0215194666bf1be7871efca3c0 is 175, key is test_row_0/A:col10/1731523148326/Put/seqid=0 2024-11-13T18:39:08,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742447_1623 (size=39949) 2024-11-13T18:39:08,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523208369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523208371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523208372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523208483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523208483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523208483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,549 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/3388f145ca074a4793b5730cdaad7dc8 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/3388f145ca074a4793b5730cdaad7dc8 2024-11-13T18:39:08,553 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into 3388f145ca074a4793b5730cdaad7dc8(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:08,553 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:08,553 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523147712; duration=0sec 2024-11-13T18:39:08,553 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:08,553 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:39:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523208693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523208693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523208693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:08,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/203d5d0215194666bf1be7871efca3c0 2024-11-13T18:39:08,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/f33d0ed118ab44d8a6512467ffa96a63 is 50, key is test_row_0/B:col10/1731523148326/Put/seqid=0 2024-11-13T18:39:08,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742448_1624 (size=12301) 2024-11-13T18:39:08,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/f33d0ed118ab44d8a6512467ffa96a63 2024-11-13T18:39:08,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/4464ea51a7794be386a4e81a7e06ba38 is 50, key is test_row_0/C:col10/1731523148326/Put/seqid=0 2024-11-13T18:39:08,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742449_1625 (size=12301) 2024-11-13T18:39:08,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/4464ea51a7794be386a4e81a7e06ba38 2024-11-13T18:39:08,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/203d5d0215194666bf1be7871efca3c0 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/203d5d0215194666bf1be7871efca3c0 2024-11-13T18:39:08,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/203d5d0215194666bf1be7871efca3c0, entries=200, sequenceid=292, filesize=39.0 K 2024-11-13T18:39:08,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/f33d0ed118ab44d8a6512467ffa96a63 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/f33d0ed118ab44d8a6512467ffa96a63 2024-11-13T18:39:08,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/f33d0ed118ab44d8a6512467ffa96a63, entries=150, sequenceid=292, filesize=12.0 K 2024-11-13T18:39:08,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/4464ea51a7794be386a4e81a7e06ba38 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/4464ea51a7794be386a4e81a7e06ba38 2024-11-13T18:39:08,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/4464ea51a7794be386a4e81a7e06ba38, entries=150, sequenceid=292, filesize=12.0 K 2024-11-13T18:39:08,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 21be930938e864d73f0be2877df0a945 in 461ms, sequenceid=292, compaction requested=true 2024-11-13T18:39:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:39:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:08,788 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:39:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:39:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:08,788 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:08,789 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103030 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:08,789 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:08,789 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:39:08,789 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:39:08,789 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:08,789 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:08,789 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/3388f145ca074a4793b5730cdaad7dc8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/17d4ace58e694da9a91f602291a18c7b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/203d5d0215194666bf1be7871efca3c0] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=100.6 K 2024-11-13T18:39:08,789 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/04a01df0fe7e44ea9ebef965d66485a8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b07eb46ba79942b993cb793e86c09574, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/f33d0ed118ab44d8a6512467ffa96a63] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=36.5 K 2024-11-13T18:39:08,789 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:08,789 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/3388f145ca074a4793b5730cdaad7dc8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/17d4ace58e694da9a91f602291a18c7b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/203d5d0215194666bf1be7871efca3c0] 2024-11-13T18:39:08,789 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 04a01df0fe7e44ea9ebef965d66485a8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731523144582 2024-11-13T18:39:08,789 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3388f145ca074a4793b5730cdaad7dc8, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731523144582 2024-11-13T18:39:08,790 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting b07eb46ba79942b993cb793e86c09574, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731523146844 2024-11-13T18:39:08,790 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17d4ace58e694da9a91f602291a18c7b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731523146844 2024-11-13T18:39:08,790 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting f33d0ed118ab44d8a6512467ffa96a63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731523148005 2024-11-13T18:39:08,790 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203d5d0215194666bf1be7871efca3c0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731523148005 2024-11-13T18:39:08,797 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:08,798 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/5bd2f993b1e148cd983b3f330cf69f69 is 50, key is test_row_0/B:col10/1731523148326/Put/seqid=0 2024-11-13T18:39:08,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742450_1626 (size=13017) 2024-11-13T18:39:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T18:39:08,803 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:08,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-13T18:39:08,806 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:08,806 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:08,807 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:08,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:39:08,817 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113e80366f3c372487a804591db63fa9add_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:08,820 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113e80366f3c372487a804591db63fa9add_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:08,820 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e80366f3c372487a804591db63fa9add_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:08,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742451_1627 (size=4469) 2024-11-13T18:39:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:39:08,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T18:39:08,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:08,961 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T18:39:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111323191f8b4be340cc95310ced26000b10_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523148371/Put/seqid=0 2024-11-13T18:39:08,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742452_1628 (size=12454) 2024-11-13T18:39:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:09,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:09,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523209021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523209021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523209027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:39:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523209133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523209133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523209144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,207 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/5bd2f993b1e148cd983b3f330cf69f69 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5bd2f993b1e148cd983b3f330cf69f69 2024-11-13T18:39:09,211 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into 5bd2f993b1e148cd983b3f330cf69f69(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:09,211 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:09,211 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523148788; duration=0sec 2024-11-13T18:39:09,211 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:09,211 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:39:09,211 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:09,212 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:09,212 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:39:09,212 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:09,212 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e4d2924fe2e8411caba130c6fc894282, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/a1f49625aa4e42f49a1389857628572e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/4464ea51a7794be386a4e81a7e06ba38] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=36.5 K 2024-11-13T18:39:09,212 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e4d2924fe2e8411caba130c6fc894282, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731523144582 2024-11-13T18:39:09,212 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f49625aa4e42f49a1389857628572e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731523146844 2024-11-13T18:39:09,212 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4464ea51a7794be386a4e81a7e06ba38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731523148005 2024-11-13T18:39:09,217 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:09,218 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/8e93ce80192e4054bf6a55ead1846a49 is 50, key is test_row_0/C:col10/1731523148326/Put/seqid=0 2024-11-13T18:39:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742453_1629 (size=13017) 2024-11-13T18:39:09,224 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#528 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:09,225 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/dc53bf9d483a4daebbeb8f7e79e62bf6 is 175, key is test_row_0/A:col10/1731523148326/Put/seqid=0 2024-11-13T18:39:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742454_1630 (size=31971) 2024-11-13T18:39:09,231 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/dc53bf9d483a4daebbeb8f7e79e62bf6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/dc53bf9d483a4daebbeb8f7e79e62bf6 2024-11-13T18:39:09,235 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into dc53bf9d483a4daebbeb8f7e79e62bf6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:09,235 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:09,235 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523148788; duration=0sec 2024-11-13T18:39:09,235 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:09,235 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:39:09,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523209343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523209343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523209353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:09,380 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111323191f8b4be340cc95310ced26000b10_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323191f8b4be340cc95310ced26000b10_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:09,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/5f556cb2a9a4415dab6538a8f2ecd3fc, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:09,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/5f556cb2a9a4415dab6538a8f2ecd3fc is 175, key is test_row_0/A:col10/1731523148371/Put/seqid=0 2024-11-13T18:39:09,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742455_1631 (size=31255) 2024-11-13T18:39:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:39:09,625 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/8e93ce80192e4054bf6a55ead1846a49 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/8e93ce80192e4054bf6a55ead1846a49 2024-11-13T18:39:09,629 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into 8e93ce80192e4054bf6a55ead1846a49(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:09,629 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:09,629 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523148788; duration=0sec 2024-11-13T18:39:09,629 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:09,629 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:39:09,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523209653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523209653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523209663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:09,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:09,790 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/5f556cb2a9a4415dab6538a8f2ecd3fc 2024-11-13T18:39:09,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c85c94e34c9742f881c64b0822c90cf7 is 50, key is test_row_0/B:col10/1731523148371/Put/seqid=0 2024-11-13T18:39:09,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742456_1632 (size=12301) 2024-11-13T18:39:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:39:10,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523210163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:10,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523210163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:10,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523210173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:10,200 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c85c94e34c9742f881c64b0822c90cf7 2024-11-13T18:39:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e0dafcc22c134570b3544407e6f0f7e6 is 50, key is test_row_0/C:col10/1731523148371/Put/seqid=0 2024-11-13T18:39:10,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742457_1633 (size=12301) 2024-11-13T18:39:10,611 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e0dafcc22c134570b3544407e6f0f7e6 2024-11-13T18:39:10,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/5f556cb2a9a4415dab6538a8f2ecd3fc as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5f556cb2a9a4415dab6538a8f2ecd3fc 2024-11-13T18:39:10,618 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5f556cb2a9a4415dab6538a8f2ecd3fc, entries=150, sequenceid=316, filesize=30.5 K 2024-11-13T18:39:10,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/c85c94e34c9742f881c64b0822c90cf7 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c85c94e34c9742f881c64b0822c90cf7 2024-11-13T18:39:10,622 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c85c94e34c9742f881c64b0822c90cf7, entries=150, sequenceid=316, filesize=12.0 K 2024-11-13T18:39:10,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/e0dafcc22c134570b3544407e6f0f7e6 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0dafcc22c134570b3544407e6f0f7e6 2024-11-13T18:39:10,626 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0dafcc22c134570b3544407e6f0f7e6, entries=150, sequenceid=316, filesize=12.0 K 2024-11-13T18:39:10,626 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 21be930938e864d73f0be2877df0a945 in 1665ms, sequenceid=316, compaction requested=false 2024-11-13T18:39:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-13T18:39:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-13T18:39:10,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-13T18:39:10,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8200 sec 2024-11-13T18:39:10,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8270 sec 2024-11-13T18:39:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T18:39:10,954 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:10,955 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-13T18:39:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:10,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:39:10,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T18:39:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:39:10,957 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:10,957 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:10,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:10,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111361c0166265e249089a4ddabc9e8f4f9b_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523149019/Put/seqid=0 2024-11-13T18:39:10,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742458_1634 (size=14994) 2024-11-13T18:39:10,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:10,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111361c0166265e249089a4ddabc9e8f4f9b_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111361c0166265e249089a4ddabc9e8f4f9b_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:10,980 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/46328132aae7465dba475874937d0d22, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:10,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/46328132aae7465dba475874937d0d22 is 175, key is test_row_0/A:col10/1731523149019/Put/seqid=0 2024-11-13T18:39:10,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742459_1635 (size=39949) 2024-11-13T18:39:10,986 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/46328132aae7465dba475874937d0d22 2024-11-13T18:39:10,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/64ccb30b19fa45ac8fa2854c58ec1a17 is 50, key is test_row_0/B:col10/1731523149019/Put/seqid=0 2024-11-13T18:39:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742460_1636 (size=12301) 2024-11-13T18:39:11,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/64ccb30b19fa45ac8fa2854c58ec1a17 2024-11-13T18:39:11,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f73b21e214f14d518c4241f38a15a152 is 50, key is test_row_0/C:col10/1731523149019/Put/seqid=0 2024-11-13T18:39:11,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742461_1637 (size=12301) 2024-11-13T18:39:11,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f73b21e214f14d518c4241f38a15a152 2024-11-13T18:39:11,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/46328132aae7465dba475874937d0d22 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/46328132aae7465dba475874937d0d22 2024-11-13T18:39:11,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/46328132aae7465dba475874937d0d22, entries=200, sequenceid=333, filesize=39.0 K 2024-11-13T18:39:11,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/64ccb30b19fa45ac8fa2854c58ec1a17 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/64ccb30b19fa45ac8fa2854c58ec1a17 2024-11-13T18:39:11,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/64ccb30b19fa45ac8fa2854c58ec1a17, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T18:39:11,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f73b21e214f14d518c4241f38a15a152 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f73b21e214f14d518c4241f38a15a152 2024-11-13T18:39:11,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523211044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523211046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f73b21e214f14d518c4241f38a15a152, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T18:39:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 21be930938e864d73f0be2877df0a945 in 96ms, sequenceid=333, compaction requested=true 2024-11-13T18:39:11,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:11,052 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:39:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:39:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:11,053 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:39:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:11,053 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:11,053 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:39:11,054 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,054 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/dc53bf9d483a4daebbeb8f7e79e62bf6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5f556cb2a9a4415dab6538a8f2ecd3fc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/46328132aae7465dba475874937d0d22] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=100.8 K 2024-11-13T18:39:11,054 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,054 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/dc53bf9d483a4daebbeb8f7e79e62bf6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5f556cb2a9a4415dab6538a8f2ecd3fc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/46328132aae7465dba475874937d0d22] 2024-11-13T18:39:11,054 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:11,054 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc53bf9d483a4daebbeb8f7e79e62bf6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731523148005 2024-11-13T18:39:11,054 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:39:11,054 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,054 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5bd2f993b1e148cd983b3f330cf69f69, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c85c94e34c9742f881c64b0822c90cf7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/64ccb30b19fa45ac8fa2854c58ec1a17] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=36.7 K 2024-11-13T18:39:11,055 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f556cb2a9a4415dab6538a8f2ecd3fc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731523148366 2024-11-13T18:39:11,055 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd2f993b1e148cd983b3f330cf69f69, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731523148005 2024-11-13T18:39:11,055 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting c85c94e34c9742f881c64b0822c90cf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731523148366 2024-11-13T18:39:11,055 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46328132aae7465dba475874937d0d22, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731523149019 2024-11-13T18:39:11,056 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ccb30b19fa45ac8fa2854c58ec1a17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731523149019 2024-11-13T18:39:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:39:11,067 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:11,069 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:11,069 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113227e0e121fde409e8ca24f80a2fd1e1d_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:11,070 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/568285c152ff464aa825682e087059cf is 50, key is test_row_0/B:col10/1731523149019/Put/seqid=0 2024-11-13T18:39:11,071 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113227e0e121fde409e8ca24f80a2fd1e1d_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:11,071 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113227e0e121fde409e8ca24f80a2fd1e1d_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:11,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742463_1639 (size=4469) 2024-11-13T18:39:11,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742462_1638 (size=13119) 2024-11-13T18:39:11,098 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/568285c152ff464aa825682e087059cf as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/568285c152ff464aa825682e087059cf 2024-11-13T18:39:11,104 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into 568285c152ff464aa825682e087059cf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:11,104 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:11,104 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523151053; duration=0sec 2024-11-13T18:39:11,104 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:11,104 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:39:11,104 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:11,106 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:11,106 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:39:11,106 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,106 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/8e93ce80192e4054bf6a55ead1846a49, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0dafcc22c134570b3544407e6f0f7e6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f73b21e214f14d518c4241f38a15a152] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=36.7 K 2024-11-13T18:39:11,106 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e93ce80192e4054bf6a55ead1846a49, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731523148005 2024-11-13T18:39:11,107 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting e0dafcc22c134570b3544407e6f0f7e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731523148366 2024-11-13T18:39:11,107 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting f73b21e214f14d518c4241f38a15a152, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731523149019 2024-11-13T18:39:11,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-13T18:39:11,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,112 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T18:39:11,114 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:11,115 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/ca34ae9160b14080b99b8c8eb85e2b20 is 50, key is test_row_0/C:col10/1731523149019/Put/seqid=0 2024-11-13T18:39:11,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137211155cebcf4953bf873095448ff166_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523151044/Put/seqid=0 2024-11-13T18:39:11,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742464_1640 (size=13119) 2024-11-13T18:39:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:11,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:11,162 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/ca34ae9160b14080b99b8c8eb85e2b20 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ca34ae9160b14080b99b8c8eb85e2b20 2024-11-13T18:39:11,169 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into ca34ae9160b14080b99b8c8eb85e2b20(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:11,169 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:11,169 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523151053; duration=0sec 2024-11-13T18:39:11,169 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:11,169 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:39:11,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742465_1641 (size=12454) 2024-11-13T18:39:11,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:11,188 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137211155cebcf4953bf873095448ff166_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137211155cebcf4953bf873095448ff166_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523211183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48584 deadline: 1731523211184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523211184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/7c797f9b288f40ca8e4a4129a4074960, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,190 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_1' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/7c797f9b288f40ca8e4a4129a4074960 is 175, key is test_row_0/A:col10/1731523151044/Put/seqid=0 2024-11-13T18:39:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523211190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48554 deadline: 1731523211193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,196 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 39e84130bbc9,35235,1731522975587 for 'test_row_2' in 21be930938e864d73f0be2877df0a945 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T18:39:11,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742466_1642 (size=31255) 2024-11-13T18:39:11,203 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/7c797f9b288f40ca8e4a4129a4074960 2024-11-13T18:39:11,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/3bf9a4e915b14a59834f0d894278d973 is 50, key is test_row_0/B:col10/1731523151044/Put/seqid=0 2024-11-13T18:39:11,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742467_1643 (size=12301) 2024-11-13T18:39:11,259 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/3bf9a4e915b14a59834f0d894278d973 2024-11-13T18:39:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:39:11,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/3ae00440172a41f6868a0c52fc30551a is 50, key is test_row_0/C:col10/1731523151044/Put/seqid=0 2024-11-13T18:39:11,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742468_1644 (size=12301) 2024-11-13T18:39:11,297 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/3ae00440172a41f6868a0c52fc30551a 2024-11-13T18:39:11,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523211293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523211294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/7c797f9b288f40ca8e4a4129a4074960 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/7c797f9b288f40ca8e4a4129a4074960 2024-11-13T18:39:11,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523211304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,310 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/7c797f9b288f40ca8e4a4129a4074960, entries=150, sequenceid=356, filesize=30.5 K 2024-11-13T18:39:11,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/3bf9a4e915b14a59834f0d894278d973 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3bf9a4e915b14a59834f0d894278d973 2024-11-13T18:39:11,317 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3bf9a4e915b14a59834f0d894278d973, entries=150, sequenceid=356, filesize=12.0 K 2024-11-13T18:39:11,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/3ae00440172a41f6868a0c52fc30551a as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3ae00440172a41f6868a0c52fc30551a 2024-11-13T18:39:11,329 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3ae00440172a41f6868a0c52fc30551a, entries=150, sequenceid=356, filesize=12.0 K 2024-11-13T18:39:11,330 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 21be930938e864d73f0be2877df0a945 in 218ms, sequenceid=356, compaction requested=false 2024-11-13T18:39:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-13T18:39:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-13T18:39:11,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-13T18:39:11,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 374 msec 2024-11-13T18:39:11,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 380 msec 2024-11-13T18:39:11,491 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#536 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:11,491 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/67d88e7049ff4e01b6fcbc27d3984b79 is 175, key is test_row_0/A:col10/1731523149019/Put/seqid=0 2024-11-13T18:39:11,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742469_1645 (size=32073) 2024-11-13T18:39:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:11,506 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T18:39:11,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T18:39:11,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111342e5faec41824f02b34e47aa96159490_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523151175/Put/seqid=0 2024-11-13T18:39:11,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742470_1646 (size=14994) 2024-11-13T18:39:11,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523211540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523211543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523211546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T18:39:11,584 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:11,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-13T18:39:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-13T18:39:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T18:39:11,586 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T18:39:11,586 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T18:39:11,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T18:39:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523211653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523211653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523211663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T18:39:11,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T18:39:11,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:11,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:11,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:11,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523211863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523211863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523211873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:11,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T18:39:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:11,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:11,901 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/67d88e7049ff4e01b6fcbc27d3984b79 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/67d88e7049ff4e01b6fcbc27d3984b79 2024-11-13T18:39:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T18:39:11,905 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into 67d88e7049ff4e01b6fcbc27d3984b79(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:11,905 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:11,905 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523151052; duration=0sec 2024-11-13T18:39:11,905 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:11,905 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:39:11,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:11,919 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111342e5faec41824f02b34e47aa96159490_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111342e5faec41824f02b34e47aa96159490_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:11,919 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/0ac09f4f78b24964b5914229646e5d8b, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:11,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/0ac09f4f78b24964b5914229646e5d8b is 175, key is test_row_0/A:col10/1731523151175/Put/seqid=0 2024-11-13T18:39:11,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742471_1647 (size=39949) 2024-11-13T18:39:12,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T18:39:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,114 INFO [Thread-2434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2434. 2024-11-13T18:39:12,114 DEBUG [Thread-2434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:12,115 INFO [Thread-2432 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2432. 2024-11-13T18:39:12,115 DEBUG [Thread-2432 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:12,115 DEBUG [Thread-2432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,115 DEBUG [Thread-2432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:12,116 DEBUG [Thread-2434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,116 DEBUG [Thread-2434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:12,117 INFO [Thread-2430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2430. 2024-11-13T18:39:12,117 DEBUG [Thread-2430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:12,117 DEBUG [Thread-2430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,117 DEBUG [Thread-2430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:12,120 INFO [Thread-2428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2428. 2024-11-13T18:39:12,120 DEBUG [Thread-2428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:12,120 INFO [Thread-2436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2436. 2024-11-13T18:39:12,120 DEBUG [Thread-2428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,120 DEBUG [Thread-2436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:12,120 DEBUG [Thread-2436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,120 DEBUG [Thread-2428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:12,120 DEBUG [Thread-2436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:12,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:12,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:12,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523212173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523212174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:12,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:12,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523212183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:12,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T18:39:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T18:39:12,336 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/0ac09f4f78b24964b5914229646e5d8b 2024-11-13T18:39:12,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/57fa47eb1aac4ec099c045a669a46b9b is 50, key is test_row_0/B:col10/1731523151175/Put/seqid=0 2024-11-13T18:39:12,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T18:39:12,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742472_1648 (size=12301) 2024-11-13T18:39:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/57fa47eb1aac4ec099c045a669a46b9b 2024-11-13T18:39:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f45efba837e44a7fbc131cd06d5b5d06 is 50, key is test_row_0/C:col10/1731523151175/Put/seqid=0 2024-11-13T18:39:12,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742473_1649 (size=12301) 2024-11-13T18:39:12,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T18:39:12,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T18:39:12,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:12,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:39:12,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48612 deadline: 1731523212683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48568 deadline: 1731523212683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:12,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T18:39:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48596 deadline: 1731523212693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 , the old value is region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T18:39:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., hostname=39e84130bbc9,35235,1731522975587, seqNum=5 because the exception is null or not the one we care about 2024-11-13T18:39:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T18:39:12,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f45efba837e44a7fbc131cd06d5b5d06 2024-11-13T18:39:12,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/0ac09f4f78b24964b5914229646e5d8b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/0ac09f4f78b24964b5914229646e5d8b 2024-11-13T18:39:12,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/0ac09f4f78b24964b5914229646e5d8b, entries=200, sequenceid=372, filesize=39.0 K 2024-11-13T18:39:12,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/57fa47eb1aac4ec099c045a669a46b9b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/57fa47eb1aac4ec099c045a669a46b9b 2024-11-13T18:39:12,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/57fa47eb1aac4ec099c045a669a46b9b, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T18:39:12,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/f45efba837e44a7fbc131cd06d5b5d06 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f45efba837e44a7fbc131cd06d5b5d06 2024-11-13T18:39:12,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f45efba837e44a7fbc131cd06d5b5d06, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T18:39:12,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 21be930938e864d73f0be2877df0a945 in 1264ms, sequenceid=372, compaction requested=true 2024-11-13T18:39:12,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:12,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T18:39:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T18:39:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21be930938e864d73f0be2877df0a945:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T18:39:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:12,771 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:12,771 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:12,771 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:12,771 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:12,771 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/A is initiating minor compaction (all files) 2024-11-13T18:39:12,771 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/B is initiating minor compaction (all files) 2024-11-13T18:39:12,771 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/B in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,771 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/A in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,771 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/568285c152ff464aa825682e087059cf, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3bf9a4e915b14a59834f0d894278d973, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/57fa47eb1aac4ec099c045a669a46b9b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=36.8 K 2024-11-13T18:39:12,772 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/67d88e7049ff4e01b6fcbc27d3984b79, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/7c797f9b288f40ca8e4a4129a4074960, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/0ac09f4f78b24964b5914229646e5d8b] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=100.9 K 2024-11-13T18:39:12,772 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,772 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. files: [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/67d88e7049ff4e01b6fcbc27d3984b79, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/7c797f9b288f40ca8e4a4129a4074960, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/0ac09f4f78b24964b5914229646e5d8b] 2024-11-13T18:39:12,772 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 568285c152ff464aa825682e087059cf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731523149019 2024-11-13T18:39:12,772 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67d88e7049ff4e01b6fcbc27d3984b79, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731523149019 2024-11-13T18:39:12,772 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf9a4e915b14a59834f0d894278d973, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731523151025 2024-11-13T18:39:12,772 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c797f9b288f40ca8e4a4129a4074960, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731523151025 2024-11-13T18:39:12,772 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 57fa47eb1aac4ec099c045a669a46b9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731523151175 2024-11-13T18:39:12,772 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ac09f4f78b24964b5914229646e5d8b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731523151175 2024-11-13T18:39:12,777 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:12,777 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#B#compaction#545 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:12,778 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/3be00fc805f5494bbd9bade3c1ca178c is 50, key is test_row_0/B:col10/1731523151175/Put/seqid=0 2024-11-13T18:39:12,778 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113aaf774005fba4114985f831d80a6ac75_21be930938e864d73f0be2877df0a945 store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:12,779 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113aaf774005fba4114985f831d80a6ac75_21be930938e864d73f0be2877df0a945, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:12,779 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aaf774005fba4114985f831d80a6ac75_21be930938e864d73f0be2877df0a945 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:12,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742474_1650 (size=13221) 2024-11-13T18:39:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742475_1651 (size=4469) 2024-11-13T18:39:12,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-13T18:39:12,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:12,808 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T18:39:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113053fb7310d4e41dea5484936a6385138_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523151545/Put/seqid=0 2024-11-13T18:39:12,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742476_1652 (size=12454) 2024-11-13T18:39:13,183 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#A#compaction#546 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:13,183 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/09bf817493004789bee1de1b91ac8089 is 175, key is test_row_0/A:col10/1731523151175/Put/seqid=0 2024-11-13T18:39:13,185 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/3be00fc805f5494bbd9bade3c1ca178c as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3be00fc805f5494bbd9bade3c1ca178c 2024-11-13T18:39:13,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742477_1653 (size=32175) 2024-11-13T18:39:13,189 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/B of 21be930938e864d73f0be2877df0a945 into 3be00fc805f5494bbd9bade3c1ca178c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:13,189 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:13,189 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/B, priority=13, startTime=1731523152771; duration=0sec 2024-11-13T18:39:13,189 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T18:39:13,189 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:B 2024-11-13T18:39:13,189 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T18:39:13,189 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/09bf817493004789bee1de1b91ac8089 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/09bf817493004789bee1de1b91ac8089 2024-11-13T18:39:13,189 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T18:39:13,189 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1541): 21be930938e864d73f0be2877df0a945/C is initiating minor compaction (all files) 2024-11-13T18:39:13,190 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21be930938e864d73f0be2877df0a945/C in TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:13,190 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ca34ae9160b14080b99b8c8eb85e2b20, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3ae00440172a41f6868a0c52fc30551a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f45efba837e44a7fbc131cd06d5b5d06] into tmpdir=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp, totalSize=36.8 K 2024-11-13T18:39:13,190 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting ca34ae9160b14080b99b8c8eb85e2b20, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731523149019 2024-11-13T18:39:13,190 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae00440172a41f6868a0c52fc30551a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731523151025 2024-11-13T18:39:13,190 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] compactions.Compactor(225): Compacting f45efba837e44a7fbc131cd06d5b5d06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731523151175 2024-11-13T18:39:13,192 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/A of 21be930938e864d73f0be2877df0a945 into 09bf817493004789bee1de1b91ac8089(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:13,192 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:13,192 INFO [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/A, priority=13, startTime=1731523152770; duration=0sec 2024-11-13T18:39:13,192 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:13,192 DEBUG [RS:0;39e84130bbc9:35235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:A 2024-11-13T18:39:13,195 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21be930938e864d73f0be2877df0a945#C#compaction#548 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:39:13,195 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/80284cda13064861a0a2c8f20a21a4bd is 50, key is test_row_0/C:col10/1731523151175/Put/seqid=0 2024-11-13T18:39:13,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742478_1654 (size=13221) 2024-11-13T18:39:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35235 {}] regionserver.HRegion(8855): Flush requested on 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:13,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. as already flushing 2024-11-13T18:39:13,204 INFO [Thread-2419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2419. 2024-11-13T18:39:13,204 DEBUG [Thread-2419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:13,204 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,205 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:13,214 INFO [Thread-2423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2423. 2024-11-13T18:39:13,214 DEBUG [Thread-2423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:13,214 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,215 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:13,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:13,218 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113053fb7310d4e41dea5484936a6385138_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113053fb7310d4e41dea5484936a6385138_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:13,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/14b3a94aed02471b8d851d995e90db29, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:13,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/14b3a94aed02471b8d851d995e90db29 is 175, key is test_row_0/A:col10/1731523151545/Put/seqid=0 2024-11-13T18:39:13,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742479_1655 (size=31255) 2024-11-13T18:39:13,602 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/80284cda13064861a0a2c8f20a21a4bd as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/80284cda13064861a0a2c8f20a21a4bd 2024-11-13T18:39:13,605 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21be930938e864d73f0be2877df0a945/C of 21be930938e864d73f0be2877df0a945 into 80284cda13064861a0a2c8f20a21a4bd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:39:13,605 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:13,605 INFO [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., storeName=21be930938e864d73f0be2877df0a945/C, priority=13, startTime=1731523152771; duration=0sec 2024-11-13T18:39:13,605 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T18:39:13,605 DEBUG [RS:0;39e84130bbc9:35235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21be930938e864d73f0be2877df0a945:C 2024-11-13T18:39:13,623 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/14b3a94aed02471b8d851d995e90db29 2024-11-13T18:39:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/5b56d6f716264381a8788c530d8d6f9d is 50, key is test_row_0/B:col10/1731523151545/Put/seqid=0 2024-11-13T18:39:13,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742480_1656 (size=12301) 2024-11-13T18:39:13,695 INFO [Thread-2417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2417. 2024-11-13T18:39:13,695 INFO [Thread-2425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2425. 2024-11-13T18:39:13,695 DEBUG [Thread-2425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:13,695 DEBUG [Thread-2417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:13,695 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,695 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,695 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,695 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:13,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:13,715 INFO [Thread-2421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2421. 2024-11-13T18:39:13,715 DEBUG [Thread-2421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T18:39:13,715 DEBUG [Thread-2421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,715 DEBUG [Thread-2421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:13,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T18:39:13,883 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:39:14,044 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/5b56d6f716264381a8788c530d8d6f9d 2024-11-13T18:39:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/d7d5c19bedaa44959c5ca5f57f4a1546 is 50, key is test_row_0/C:col10/1731523151545/Put/seqid=0 2024-11-13T18:39:14,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742481_1657 (size=12301) 2024-11-13T18:39:14,454 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/d7d5c19bedaa44959c5ca5f57f4a1546 2024-11-13T18:39:14,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/14b3a94aed02471b8d851d995e90db29 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/14b3a94aed02471b8d851d995e90db29 2024-11-13T18:39:14,461 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/14b3a94aed02471b8d851d995e90db29, entries=150, sequenceid=394, filesize=30.5 K 2024-11-13T18:39:14,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/5b56d6f716264381a8788c530d8d6f9d as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5b56d6f716264381a8788c530d8d6f9d 2024-11-13T18:39:14,464 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5b56d6f716264381a8788c530d8d6f9d, entries=150, sequenceid=394, filesize=12.0 K 2024-11-13T18:39:14,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/d7d5c19bedaa44959c5ca5f57f4a1546 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/d7d5c19bedaa44959c5ca5f57f4a1546 2024-11-13T18:39:14,468 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/d7d5c19bedaa44959c5ca5f57f4a1546, entries=150, sequenceid=394, filesize=12.0 K 2024-11-13T18:39:14,468 INFO [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=33.54 KB/34350 for 21be930938e864d73f0be2877df0a945 in 1660ms, sequenceid=394, compaction requested=false 2024-11-13T18:39:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 21be930938e864d73f0be2877df0a945: 2024-11-13T18:39:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-13T18:39:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-13T18:39:14,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-13T18:39:14,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8830 sec 2024-11-13T18:39:14,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.8870 sec 2024-11-13T18:39:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T18:39:15,743 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2673 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2655 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2636 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2684 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2658 2024-11-13T18:39:15,743 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T18:39:15,744 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:39:15,744 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:39:15,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:15,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:15,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:15,744 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-13T18:39:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T18:39:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T18:39:15,747 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523155747"}]},"ts":"1731523155747"} 2024-11-13T18:39:15,748 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T18:39:15,748 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T18:39:15,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T18:39:15,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, UNASSIGN}] 2024-11-13T18:39:15,750 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, UNASSIGN 2024-11-13T18:39:15,751 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=21be930938e864d73f0be2877df0a945, regionState=CLOSING, regionLocation=39e84130bbc9,35235,1731522975587 2024-11-13T18:39:15,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, UNASSIGN because future has completed 2024-11-13T18:39:15,752 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:39:15,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587}] 2024-11-13T18:39:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T18:39:15,904 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:15,904 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:39:15,904 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 21be930938e864d73f0be2877df0a945, disabling compactions & flushes 2024-11-13T18:39:15,904 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:15,904 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:15,904 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. after waiting 0 ms 2024-11-13T18:39:15,904 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:15,904 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 21be930938e864d73f0be2877df0a945 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T18:39:15,909 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131c009928d93048ca976dca1c11a7a774_21be930938e864d73f0be2877df0a945 is 50, key is test_row_0/A:col10/1731523153714/Put/seqid=0 2024-11-13T18:39:15,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742482_1658 (size=12454) 2024-11-13T18:39:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T18:39:16,312 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:39:16,315 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131c009928d93048ca976dca1c11a7a774_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c009928d93048ca976dca1c11a7a774_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:16,316 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/d696e8693d4b4e13a2e242730c02fd0b, store: [table=TestAcidGuarantees family=A region=21be930938e864d73f0be2877df0a945] 2024-11-13T18:39:16,317 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/d696e8693d4b4e13a2e242730c02fd0b is 175, key is test_row_0/A:col10/1731523153714/Put/seqid=0 2024-11-13T18:39:16,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742483_1659 (size=31255) 2024-11-13T18:39:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T18:39:16,720 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/d696e8693d4b4e13a2e242730c02fd0b 2024-11-13T18:39:16,725 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/be35c7108bfc4d698c40fd002478c998 is 50, key is test_row_0/B:col10/1731523153714/Put/seqid=0 2024-11-13T18:39:16,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742484_1660 (size=12301) 2024-11-13T18:39:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T18:39:17,129 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/be35c7108bfc4d698c40fd002478c998 2024-11-13T18:39:17,134 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/d76409deaf4c4528a1b15ca211656a48 is 50, key is test_row_0/C:col10/1731523153714/Put/seqid=0 2024-11-13T18:39:17,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742485_1661 (size=12301) 2024-11-13T18:39:17,137 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/d76409deaf4c4528a1b15ca211656a48 2024-11-13T18:39:17,140 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/A/d696e8693d4b4e13a2e242730c02fd0b as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d696e8693d4b4e13a2e242730c02fd0b 2024-11-13T18:39:17,143 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d696e8693d4b4e13a2e242730c02fd0b, entries=150, sequenceid=405, filesize=30.5 K 2024-11-13T18:39:17,143 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/B/be35c7108bfc4d698c40fd002478c998 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/be35c7108bfc4d698c40fd002478c998 2024-11-13T18:39:17,146 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/be35c7108bfc4d698c40fd002478c998, entries=150, sequenceid=405, filesize=12.0 K 2024-11-13T18:39:17,146 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/.tmp/C/d76409deaf4c4528a1b15ca211656a48 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/d76409deaf4c4528a1b15ca211656a48 2024-11-13T18:39:17,148 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/d76409deaf4c4528a1b15ca211656a48, entries=150, sequenceid=405, filesize=12.0 K 2024-11-13T18:39:17,149 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 21be930938e864d73f0be2877df0a945 in 1244ms, sequenceid=405, compaction requested=true 2024-11-13T18:39:17,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f2fb258e664624a75a608af63417c9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/386486a82a2849b4bebc8b501d660262, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1f1686b6a9084dd3bd3de0c803db68cb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bf737576d82949d7be5e24a499fab584, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53ff9078c54a4e4580a72fe8f7a97b8a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53e5a3eaabe64376900dc945f17af8af, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fe5dfc94d9dc4fba943be238dbdfef5e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/edf1bea2774146a69f923d9f3f21ab4c, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/52995ec92c3b41a2ad95abeae507d060, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/c3069290613b411ca678115058a13b9b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1a0b2672aaef49eb81f938ef40f83961, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bcfc0e2378c649b1ac741f49bb8381c6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5bee6beeffdb43af8a9b69b0cf53d645, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/800005fa980142798a3ed54062cee20f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f8c7d5542b4c2e8cf0140b8753dcf8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/6a31d28054ce495f9030a60a7dbfc4ba, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fdb4284ce5d74d4ba260b69521003c8e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d10a3b97e83a4ab0b91ef682f60a61f5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/3388f145ca074a4793b5730cdaad7dc8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/17d4ace58e694da9a91f602291a18c7b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/203d5d0215194666bf1be7871efca3c0, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/dc53bf9d483a4daebbeb8f7e79e62bf6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5f556cb2a9a4415dab6538a8f2ecd3fc, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/46328132aae7465dba475874937d0d22, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/67d88e7049ff4e01b6fcbc27d3984b79, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/7c797f9b288f40ca8e4a4129a4074960, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/0ac09f4f78b24964b5914229646e5d8b] to archive 2024-11-13T18:39:17,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:39:17,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f2fb258e664624a75a608af63417c9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f2fb258e664624a75a608af63417c9 2024-11-13T18:39:17,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/386486a82a2849b4bebc8b501d660262 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/386486a82a2849b4bebc8b501d660262 2024-11-13T18:39:17,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1f1686b6a9084dd3bd3de0c803db68cb to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1f1686b6a9084dd3bd3de0c803db68cb 2024-11-13T18:39:17,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bf737576d82949d7be5e24a499fab584 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bf737576d82949d7be5e24a499fab584 2024-11-13T18:39:17,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53ff9078c54a4e4580a72fe8f7a97b8a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53ff9078c54a4e4580a72fe8f7a97b8a 2024-11-13T18:39:17,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53e5a3eaabe64376900dc945f17af8af to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/53e5a3eaabe64376900dc945f17af8af 2024-11-13T18:39:17,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fe5dfc94d9dc4fba943be238dbdfef5e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fe5dfc94d9dc4fba943be238dbdfef5e 2024-11-13T18:39:17,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/edf1bea2774146a69f923d9f3f21ab4c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/edf1bea2774146a69f923d9f3f21ab4c 2024-11-13T18:39:17,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/52995ec92c3b41a2ad95abeae507d060 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/52995ec92c3b41a2ad95abeae507d060 2024-11-13T18:39:17,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/c3069290613b411ca678115058a13b9b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/c3069290613b411ca678115058a13b9b 2024-11-13T18:39:17,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1a0b2672aaef49eb81f938ef40f83961 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/1a0b2672aaef49eb81f938ef40f83961 2024-11-13T18:39:17,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bcfc0e2378c649b1ac741f49bb8381c6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/bcfc0e2378c649b1ac741f49bb8381c6 2024-11-13T18:39:17,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5bee6beeffdb43af8a9b69b0cf53d645 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5bee6beeffdb43af8a9b69b0cf53d645 2024-11-13T18:39:17,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/800005fa980142798a3ed54062cee20f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/800005fa980142798a3ed54062cee20f 2024-11-13T18:39:17,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f8c7d5542b4c2e8cf0140b8753dcf8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/79f8c7d5542b4c2e8cf0140b8753dcf8 2024-11-13T18:39:17,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/6a31d28054ce495f9030a60a7dbfc4ba to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/6a31d28054ce495f9030a60a7dbfc4ba 2024-11-13T18:39:17,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fdb4284ce5d74d4ba260b69521003c8e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/fdb4284ce5d74d4ba260b69521003c8e 2024-11-13T18:39:17,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d10a3b97e83a4ab0b91ef682f60a61f5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d10a3b97e83a4ab0b91ef682f60a61f5 2024-11-13T18:39:17,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/3388f145ca074a4793b5730cdaad7dc8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/3388f145ca074a4793b5730cdaad7dc8 2024-11-13T18:39:17,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/17d4ace58e694da9a91f602291a18c7b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/17d4ace58e694da9a91f602291a18c7b 2024-11-13T18:39:17,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/203d5d0215194666bf1be7871efca3c0 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/203d5d0215194666bf1be7871efca3c0 2024-11-13T18:39:17,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/dc53bf9d483a4daebbeb8f7e79e62bf6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/dc53bf9d483a4daebbeb8f7e79e62bf6 2024-11-13T18:39:17,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5f556cb2a9a4415dab6538a8f2ecd3fc to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/5f556cb2a9a4415dab6538a8f2ecd3fc 2024-11-13T18:39:17,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/46328132aae7465dba475874937d0d22 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/46328132aae7465dba475874937d0d22 2024-11-13T18:39:17,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/67d88e7049ff4e01b6fcbc27d3984b79 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/67d88e7049ff4e01b6fcbc27d3984b79 2024-11-13T18:39:17,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/7c797f9b288f40ca8e4a4129a4074960 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/7c797f9b288f40ca8e4a4129a4074960 2024-11-13T18:39:17,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/0ac09f4f78b24964b5914229646e5d8b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/0ac09f4f78b24964b5914229646e5d8b 2024-11-13T18:39:17,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3b435627f8cb4ab1b53bca26062c9558, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b7e4c9526ada43d88950db2d422b60b9, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/7ce6b44543374a9192399207e99d5db2, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/2801dbe1521a42068b3afac4f2415836, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bbef9e4f4592440d83bc6aec2cf8e7f3, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/984916fc634946daa8a88b2e228f205b, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/cd4951b6a0b4404392cce78d2301d7bd, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/a09389fab5f6480188898e802adfd56a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/76e8b866886c405db228dbc3190e146e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/d09b7e14fa0140fe986f4abc587760f5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/79b2dc1965ea43d0947b1d028e74e935, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/429d4314867944e3b951689408be9e81, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c1c5314e2a0b480c95a25a0741a16f2a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bd65d810bcd8459dbd8b1fdd2904ee0f, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/dac2e0a33c87408d8b092e846a9f9290, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/717a3310294641acaba928ae64a54b72, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/31f271454d0941e78f729667c62fafcb, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/04a01df0fe7e44ea9ebef965d66485a8, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c191edc170574ea58c48c51fafbd1355, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b07eb46ba79942b993cb793e86c09574, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5bd2f993b1e148cd983b3f330cf69f69, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/f33d0ed118ab44d8a6512467ffa96a63, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c85c94e34c9742f881c64b0822c90cf7, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/568285c152ff464aa825682e087059cf, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/64ccb30b19fa45ac8fa2854c58ec1a17, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3bf9a4e915b14a59834f0d894278d973, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/57fa47eb1aac4ec099c045a669a46b9b] to archive 2024-11-13T18:39:17,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:39:17,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3b435627f8cb4ab1b53bca26062c9558 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3b435627f8cb4ab1b53bca26062c9558 2024-11-13T18:39:17,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b7e4c9526ada43d88950db2d422b60b9 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b7e4c9526ada43d88950db2d422b60b9 2024-11-13T18:39:17,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/7ce6b44543374a9192399207e99d5db2 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/7ce6b44543374a9192399207e99d5db2 2024-11-13T18:39:17,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/2801dbe1521a42068b3afac4f2415836 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/2801dbe1521a42068b3afac4f2415836 2024-11-13T18:39:17,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bbef9e4f4592440d83bc6aec2cf8e7f3 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bbef9e4f4592440d83bc6aec2cf8e7f3 2024-11-13T18:39:17,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/984916fc634946daa8a88b2e228f205b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/984916fc634946daa8a88b2e228f205b 2024-11-13T18:39:17,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/cd4951b6a0b4404392cce78d2301d7bd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/cd4951b6a0b4404392cce78d2301d7bd 2024-11-13T18:39:17,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/a09389fab5f6480188898e802adfd56a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/a09389fab5f6480188898e802adfd56a 2024-11-13T18:39:17,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/76e8b866886c405db228dbc3190e146e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/76e8b866886c405db228dbc3190e146e 2024-11-13T18:39:17,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/d09b7e14fa0140fe986f4abc587760f5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/d09b7e14fa0140fe986f4abc587760f5 2024-11-13T18:39:17,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/79b2dc1965ea43d0947b1d028e74e935 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/79b2dc1965ea43d0947b1d028e74e935 2024-11-13T18:39:17,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/429d4314867944e3b951689408be9e81 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/429d4314867944e3b951689408be9e81 2024-11-13T18:39:17,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c1c5314e2a0b480c95a25a0741a16f2a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c1c5314e2a0b480c95a25a0741a16f2a 2024-11-13T18:39:17,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bd65d810bcd8459dbd8b1fdd2904ee0f to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/bd65d810bcd8459dbd8b1fdd2904ee0f 2024-11-13T18:39:17,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/dac2e0a33c87408d8b092e846a9f9290 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/dac2e0a33c87408d8b092e846a9f9290 2024-11-13T18:39:17,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/717a3310294641acaba928ae64a54b72 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/717a3310294641acaba928ae64a54b72 2024-11-13T18:39:17,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/31f271454d0941e78f729667c62fafcb to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/31f271454d0941e78f729667c62fafcb 2024-11-13T18:39:17,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/04a01df0fe7e44ea9ebef965d66485a8 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/04a01df0fe7e44ea9ebef965d66485a8 2024-11-13T18:39:17,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c191edc170574ea58c48c51fafbd1355 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c191edc170574ea58c48c51fafbd1355 2024-11-13T18:39:17,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b07eb46ba79942b993cb793e86c09574 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/b07eb46ba79942b993cb793e86c09574 2024-11-13T18:39:17,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5bd2f993b1e148cd983b3f330cf69f69 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5bd2f993b1e148cd983b3f330cf69f69 2024-11-13T18:39:17,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/f33d0ed118ab44d8a6512467ffa96a63 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/f33d0ed118ab44d8a6512467ffa96a63 2024-11-13T18:39:17,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c85c94e34c9742f881c64b0822c90cf7 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/c85c94e34c9742f881c64b0822c90cf7 2024-11-13T18:39:17,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/568285c152ff464aa825682e087059cf to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/568285c152ff464aa825682e087059cf 2024-11-13T18:39:17,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/64ccb30b19fa45ac8fa2854c58ec1a17 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/64ccb30b19fa45ac8fa2854c58ec1a17 2024-11-13T18:39:17,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3bf9a4e915b14a59834f0d894278d973 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3bf9a4e915b14a59834f0d894278d973 2024-11-13T18:39:17,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/57fa47eb1aac4ec099c045a669a46b9b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/57fa47eb1aac4ec099c045a669a46b9b 2024-11-13T18:39:17,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/2e216adf21d3470a89f4bea3e78e5319, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e59d7c333b7348cb947360efb41450d6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/56ed842c5cc34dc9a91244b45a156ed1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3b601b0e58d14c5d86a9c46fbc906a11, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ce915e4cdd5a410593ea01283b0ae663, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0e692f36673430abece9d8d8391dc0e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/b4228241de924354bb481a3146a07e98, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/52f6ed441bdd4c35a284140cc3b08c45, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/446f18683de74b4db6554979aeb244c5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/1fe9697a87984642a2a8d39071c50fb5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/bfb1550f6d674e9ebddeb43445779af1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/375edf0bd3444d8dacac0d0c713dd5af, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/9b3c696f33a14f0bb41b7986ccb3b686, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/0135be243fed4852ad1b461e0c5944be, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/74bbed2c03f7422598932c4860b43d77, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/12eac78505ad4758b848e58287dcfed1, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/cf51c6f8cf4c4539b555f6648c4c26b5, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e4d2924fe2e8411caba130c6fc894282, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f6c53a4269e9441ab3a2d2539349a719, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/a1f49625aa4e42f49a1389857628572e, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/8e93ce80192e4054bf6a55ead1846a49, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/4464ea51a7794be386a4e81a7e06ba38, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0dafcc22c134570b3544407e6f0f7e6, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ca34ae9160b14080b99b8c8eb85e2b20, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f73b21e214f14d518c4241f38a15a152, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3ae00440172a41f6868a0c52fc30551a, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f45efba837e44a7fbc131cd06d5b5d06] to archive 2024-11-13T18:39:17,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:39:17,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/2e216adf21d3470a89f4bea3e78e5319 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/2e216adf21d3470a89f4bea3e78e5319 2024-11-13T18:39:17,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e59d7c333b7348cb947360efb41450d6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e59d7c333b7348cb947360efb41450d6 2024-11-13T18:39:17,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/56ed842c5cc34dc9a91244b45a156ed1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/56ed842c5cc34dc9a91244b45a156ed1 2024-11-13T18:39:17,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3b601b0e58d14c5d86a9c46fbc906a11 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3b601b0e58d14c5d86a9c46fbc906a11 2024-11-13T18:39:17,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ce915e4cdd5a410593ea01283b0ae663 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ce915e4cdd5a410593ea01283b0ae663 2024-11-13T18:39:17,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0e692f36673430abece9d8d8391dc0e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0e692f36673430abece9d8d8391dc0e 2024-11-13T18:39:17,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/b4228241de924354bb481a3146a07e98 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/b4228241de924354bb481a3146a07e98 2024-11-13T18:39:17,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/52f6ed441bdd4c35a284140cc3b08c45 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/52f6ed441bdd4c35a284140cc3b08c45 2024-11-13T18:39:17,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/446f18683de74b4db6554979aeb244c5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/446f18683de74b4db6554979aeb244c5 2024-11-13T18:39:17,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/1fe9697a87984642a2a8d39071c50fb5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/1fe9697a87984642a2a8d39071c50fb5 2024-11-13T18:39:17,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/bfb1550f6d674e9ebddeb43445779af1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/bfb1550f6d674e9ebddeb43445779af1 2024-11-13T18:39:17,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/375edf0bd3444d8dacac0d0c713dd5af to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/375edf0bd3444d8dacac0d0c713dd5af 2024-11-13T18:39:17,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/9b3c696f33a14f0bb41b7986ccb3b686 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/9b3c696f33a14f0bb41b7986ccb3b686 2024-11-13T18:39:17,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/0135be243fed4852ad1b461e0c5944be to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/0135be243fed4852ad1b461e0c5944be 2024-11-13T18:39:17,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/74bbed2c03f7422598932c4860b43d77 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/74bbed2c03f7422598932c4860b43d77 2024-11-13T18:39:17,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/12eac78505ad4758b848e58287dcfed1 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/12eac78505ad4758b848e58287dcfed1 2024-11-13T18:39:17,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/cf51c6f8cf4c4539b555f6648c4c26b5 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/cf51c6f8cf4c4539b555f6648c4c26b5 2024-11-13T18:39:17,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e4d2924fe2e8411caba130c6fc894282 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e4d2924fe2e8411caba130c6fc894282 2024-11-13T18:39:17,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f6c53a4269e9441ab3a2d2539349a719 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f6c53a4269e9441ab3a2d2539349a719 2024-11-13T18:39:17,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/a1f49625aa4e42f49a1389857628572e to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/a1f49625aa4e42f49a1389857628572e 2024-11-13T18:39:17,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/8e93ce80192e4054bf6a55ead1846a49 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/8e93ce80192e4054bf6a55ead1846a49 2024-11-13T18:39:17,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/4464ea51a7794be386a4e81a7e06ba38 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/4464ea51a7794be386a4e81a7e06ba38 2024-11-13T18:39:17,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0dafcc22c134570b3544407e6f0f7e6 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/e0dafcc22c134570b3544407e6f0f7e6 2024-11-13T18:39:17,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ca34ae9160b14080b99b8c8eb85e2b20 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/ca34ae9160b14080b99b8c8eb85e2b20 2024-11-13T18:39:17,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f73b21e214f14d518c4241f38a15a152 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f73b21e214f14d518c4241f38a15a152 2024-11-13T18:39:17,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3ae00440172a41f6868a0c52fc30551a to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/3ae00440172a41f6868a0c52fc30551a 2024-11-13T18:39:17,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f45efba837e44a7fbc131cd06d5b5d06 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/f45efba837e44a7fbc131cd06d5b5d06 2024-11-13T18:39:17,215 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/recovered.edits/408.seqid, newMaxSeqId=408, maxSeqId=4 2024-11-13T18:39:17,215 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. 2024-11-13T18:39:17,215 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 21be930938e864d73f0be2877df0a945: Waiting for close lock at 1731523155904Running coprocessor pre-close hooks at 1731523155904Disabling compacts and flushes for region at 1731523155904Disabling writes for close at 1731523155904Obtaining lock to block concurrent updates at 1731523155904Preparing flush snapshotting stores in 21be930938e864d73f0be2877df0a945 at 1731523155904Finished memstore snapshotting TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731523155904Flushing stores of TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945. at 1731523155905 (+1 ms)Flushing 21be930938e864d73f0be2877df0a945/A: creating writer at 1731523155905Flushing mob file 21be930938e864d73f0be2877df0a945/A: appending metadata at 1731523155909 (+4 ms)Flushing mob file 21be930938e864d73f0be2877df0a945/A: closing flushed file at 1731523155909Flushing 21be930938e864d73f0be2877df0a945/A: appending metadata at 1731523156316 (+407 ms)Flushing 21be930938e864d73f0be2877df0a945/A: closing flushed file at 1731523156316Flushing 21be930938e864d73f0be2877df0a945/B: creating writer at 1731523156723 (+407 ms)Flushing 21be930938e864d73f0be2877df0a945/B: appending metadata at 1731523156725 (+2 ms)Flushing 21be930938e864d73f0be2877df0a945/B: closing flushed file at 1731523156725Flushing 21be930938e864d73f0be2877df0a945/C: creating writer at 1731523157132 (+407 ms)Flushing 21be930938e864d73f0be2877df0a945/C: appending metadata at 1731523157133 (+1 ms)Flushing 21be930938e864d73f0be2877df0a945/C: closing flushed file at 1731523157134 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2033fd11: reopening flushed file at 1731523157140 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@957ed4b: reopening flushed file at 1731523157143 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b735493: reopening flushed file at 1731523157146 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 21be930938e864d73f0be2877df0a945 in 1244ms, sequenceid=405, compaction requested=true at 1731523157149 (+3 ms)Writing region close event to WAL at 1731523157212 (+63 ms)Running coprocessor post-close hooks at 1731523157215 (+3 ms)Closed at 1731523157215 2024-11-13T18:39:17,217 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,217 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=21be930938e864d73f0be2877df0a945, regionState=CLOSED 2024-11-13T18:39:17,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 because future has completed 2024-11-13T18:39:17,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-13T18:39:17,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 21be930938e864d73f0be2877df0a945, server=39e84130bbc9,35235,1731522975587 in 1.4670 sec 2024-11-13T18:39:17,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-13T18:39:17,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21be930938e864d73f0be2877df0a945, UNASSIGN in 1.4700 sec 2024-11-13T18:39:17,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-13T18:39:17,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4730 sec 2024-11-13T18:39:17,223 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731523157223"}]},"ts":"1731523157223"} 2024-11-13T18:39:17,224 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T18:39:17,224 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T18:39:17,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-13T18:39:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-13T18:39:17,894 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:17,894 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-13T18:39:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:39:17,896 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:39:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-13T18:39:17,896 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:39:17,898 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,899 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C, FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/recovered.edits] 2024-11-13T18:39:17,901 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/09bf817493004789bee1de1b91ac8089 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/09bf817493004789bee1de1b91ac8089 2024-11-13T18:39:17,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/14b3a94aed02471b8d851d995e90db29 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/14b3a94aed02471b8d851d995e90db29 2024-11-13T18:39:17,903 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d696e8693d4b4e13a2e242730c02fd0b to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/A/d696e8693d4b4e13a2e242730c02fd0b 2024-11-13T18:39:17,904 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3be00fc805f5494bbd9bade3c1ca178c to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/3be00fc805f5494bbd9bade3c1ca178c 2024-11-13T18:39:17,905 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5b56d6f716264381a8788c530d8d6f9d to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/5b56d6f716264381a8788c530d8d6f9d 2024-11-13T18:39:17,906 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/be35c7108bfc4d698c40fd002478c998 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/B/be35c7108bfc4d698c40fd002478c998 2024-11-13T18:39:17,907 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/80284cda13064861a0a2c8f20a21a4bd to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/80284cda13064861a0a2c8f20a21a4bd 2024-11-13T18:39:17,908 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/d76409deaf4c4528a1b15ca211656a48 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/d76409deaf4c4528a1b15ca211656a48 2024-11-13T18:39:17,909 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/d7d5c19bedaa44959c5ca5f57f4a1546 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/C/d7d5c19bedaa44959c5ca5f57f4a1546 2024-11-13T18:39:17,911 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/recovered.edits/408.seqid to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945/recovered.edits/408.seqid 2024-11-13T18:39:17,911 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/default/TestAcidGuarantees/21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,911 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T18:39:17,912 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:39:17,912 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T18:39:17,914 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113053fb7310d4e41dea5484936a6385138_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113053fb7310d4e41dea5484936a6385138_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,915 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130adbf28658434d36956295da1e98ced7_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130adbf28658434d36956295da1e98ced7_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,915 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130e0314d515af4fb8a9fd45c95b6e18b3_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130e0314d515af4fb8a9fd45c95b6e18b3_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,916 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c009928d93048ca976dca1c11a7a774_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c009928d93048ca976dca1c11a7a774_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,917 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111321caf8b1022e4a6cb11d0f567251d594_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111321caf8b1022e4a6cb11d0f567251d594_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,918 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323191f8b4be340cc95310ced26000b10_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111323191f8b4be340cc95310ced26000b10_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,919 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d2710a7bab54c99b01fe26ee1b44b73_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d2710a7bab54c99b01fe26ee1b44b73_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,920 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d5b34b8441d4f5b9fe11f7184f1ac9e_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132d5b34b8441d4f5b9fe11f7184f1ac9e_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,920 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f78dd80b55645329d25effee6a37b1a_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f78dd80b55645329d25effee6a37b1a_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,921 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111342e5faec41824f02b34e47aa96159490_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111342e5faec41824f02b34e47aa96159490_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,922 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134648b51cb6fb4a718e82f459c9fbd554_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134648b51cb6fb4a718e82f459c9fbd554_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,923 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113498548f535104ac79a72051d771fe4f5_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113498548f535104ac79a72051d771fe4f5_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,924 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111361c0166265e249089a4ddabc9e8f4f9b_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111361c0166265e249089a4ddabc9e8f4f9b_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,924 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137211155cebcf4953bf873095448ff166_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137211155cebcf4953bf873095448ff166_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,925 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383515a60190c4968966350c1aa797d63_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383515a60190c4968966350c1aa797d63_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,926 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a1bf7500e7624f51a646a521263f905f_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a1bf7500e7624f51a646a521263f905f_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,927 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a680374169b6495c8e9f795c33328010_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a680374169b6495c8e9f795c33328010_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,928 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1a0e5b3bce646a59f9e9c53c38de578_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1a0e5b3bce646a59f9e9c53c38de578_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,928 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d9e86dea17564cc6be1e0b26691cde29_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d9e86dea17564cc6be1e0b26691cde29_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,929 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f237354b4df64a9281828dbb0f65520a_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f237354b4df64a9281828dbb0f65520a_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,930 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ffaf53e3d61d409a8ca95f246ea53895_21be930938e864d73f0be2877df0a945 to hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ffaf53e3d61d409a8ca95f246ea53895_21be930938e864d73f0be2877df0a945 2024-11-13T18:39:17,931 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T18:39:17,932 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:39:17,935 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T18:39:17,936 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T18:39:17,937 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:39:17,937 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T18:39:17,937 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731523157937"}]},"ts":"9223372036854775807"} 2024-11-13T18:39:17,939 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T18:39:17,939 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 21be930938e864d73f0be2877df0a945, NAME => 'TestAcidGuarantees,,1731523129012.21be930938e864d73f0be2877df0a945.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T18:39:17,939 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T18:39:17,939 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731523157939"}]},"ts":"9223372036854775807"} 2024-11-13T18:39:17,940 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T18:39:17,941 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T18:39:17,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-13T18:39:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36105 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-13T18:39:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T18:39:18,004 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T18:39:18,013 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 242), OpenFileDescriptor=457 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=770 (was 1069), ProcessCount=11 (was 11), AvailableMemoryMB=7568 (was 7764) 2024-11-13T18:39:18,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T18:39:18,013 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:39:18,013 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:39:18,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:18,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:18,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:39:18,013 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T18:39:18,013 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=129225742, stopped=false 2024-11-13T18:39:18,014 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=39e84130bbc9,36105,1731522974847 2024-11-13T18:39:18,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:39:18,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:39:18,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:39:18,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:39:18,016 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:39:18,016 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:39:18,016 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:39:18,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:18,016 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:39:18,017 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:39:18,017 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39e84130bbc9,35235,1731522975587' ***** 2024-11-13T18:39:18,017 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:39:18,017 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:39:18,017 INFO [RS:0;39e84130bbc9:35235 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:39:18,017 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:39:18,018 INFO [RS:0;39e84130bbc9:35235 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:39:18,018 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(959): stopping server 39e84130bbc9,35235,1731522975587 2024-11-13T18:39:18,018 INFO [RS:0;39e84130bbc9:35235 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:39:18,018 INFO [RS:0;39e84130bbc9:35235 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;39e84130bbc9:35235. 2024-11-13T18:39:18,018 DEBUG [RS:0;39e84130bbc9:35235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:39:18,018 DEBUG [RS:0;39e84130bbc9:35235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:18,018 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:39:18,018 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:39:18,018 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:39:18,018 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:39:18,019 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T18:39:18,019 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T18:39:18,019 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:18,019 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:39:18,019 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:39:18,019 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:39:18,019 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:39:18,019 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:39:18,019 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T18:39:18,043 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/info/16cbc2c5f2df44d4b0d5f252c830e555 is 95, key is TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681./info:/1731523006049/DeleteFamily/seqid=0 2024-11-13T18:39:18,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742486_1662 (size=6514) 2024-11-13T18:39:18,064 INFO [regionserver/39e84130bbc9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:39:18,219 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:18,367 INFO [regionserver/39e84130bbc9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T18:39:18,367 INFO [regionserver/39e84130bbc9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T18:39:18,419 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:18,448 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/info/16cbc2c5f2df44d4b0d5f252c830e555 2024-11-13T18:39:18,466 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/ns/3a9aa5de4243419b8863e90f310c5f97 is 93, key is TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681./ns:/1731523005608/DeleteFamily/seqid=0 2024-11-13T18:39:18,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742487_1663 (size=5906) 2024-11-13T18:39:18,620 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:18,820 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:18,869 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/ns/3a9aa5de4243419b8863e90f310c5f97 2024-11-13T18:39:18,886 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/rep_barrier/c50175bd00764ba5afe023c9fa542d85 is 102, key is TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681./rep_barrier:/1731523005608/DeleteFamily/seqid=0 2024-11-13T18:39:18,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742488_1664 (size=6025) 2024-11-13T18:39:19,020 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:19,220 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:19,289 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/rep_barrier/c50175bd00764ba5afe023c9fa542d85 2024-11-13T18:39:19,313 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/table/f5467392aa3742d6a38879d13e82b849 is 96, key is TestAcidGuarantees,,1731522978449.430b0529634d320a32ba155ab3165681./table:/1731523005608/DeleteFamily/seqid=0 2024-11-13T18:39:19,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742489_1665 (size=5936) 2024-11-13T18:39:19,420 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:19,621 DEBUG [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T18:39:19,720 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/table/f5467392aa3742d6a38879d13e82b849 2024-11-13T18:39:19,723 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/info/16cbc2c5f2df44d4b0d5f252c830e555 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/info/16cbc2c5f2df44d4b0d5f252c830e555 2024-11-13T18:39:19,726 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/info/16cbc2c5f2df44d4b0d5f252c830e555, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T18:39:19,726 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/ns/3a9aa5de4243419b8863e90f310c5f97 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/ns/3a9aa5de4243419b8863e90f310c5f97 2024-11-13T18:39:19,728 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/ns/3a9aa5de4243419b8863e90f310c5f97, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T18:39:19,729 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/rep_barrier/c50175bd00764ba5afe023c9fa542d85 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/rep_barrier/c50175bd00764ba5afe023c9fa542d85 2024-11-13T18:39:19,731 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/rep_barrier/c50175bd00764ba5afe023c9fa542d85, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T18:39:19,731 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/.tmp/table/f5467392aa3742d6a38879d13e82b849 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/table/f5467392aa3742d6a38879d13e82b849 2024-11-13T18:39:19,734 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/table/f5467392aa3742d6a38879d13e82b849, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T18:39:19,734 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1715ms, sequenceid=90, compaction requested=false 2024-11-13T18:39:19,738 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T18:39:19,739 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:39:19,739 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:39:19,739 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731523158019Running coprocessor pre-close hooks at 1731523158019Disabling compacts and flushes for region at 1731523158019Disabling writes for close at 1731523158019Obtaining lock to block concurrent updates at 1731523158019Preparing flush snapshotting stores in 1588230740 at 1731523158019Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731523158019Flushing stores of hbase:meta,,1.1588230740 at 1731523158020 (+1 ms)Flushing 1588230740/info: creating writer at 1731523158020Flushing 1588230740/info: appending metadata at 1731523158043 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731523158043Flushing 1588230740/ns: creating writer at 1731523158451 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731523158465 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731523158465Flushing 1588230740/rep_barrier: creating writer at 1731523158872 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731523158886 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731523158886Flushing 1588230740/table: creating writer at 1731523159292 (+406 ms)Flushing 1588230740/table: appending metadata at 1731523159312 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731523159312Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2740b346: reopening flushed file at 1731523159723 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4023744c: reopening flushed file at 1731523159726 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d71b2e3: reopening flushed file at 1731523159729 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a8e0a3: reopening flushed file at 1731523159731 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1715ms, sequenceid=90, compaction requested=false at 1731523159734 (+3 ms)Writing region close event to WAL at 1731523159736 (+2 ms)Running coprocessor post-close hooks at 1731523159739 (+3 ms)Closed at 1731523159739 2024-11-13T18:39:19,739 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:39:19,821 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(976): stopping server 39e84130bbc9,35235,1731522975587; all regions closed. 2024-11-13T18:39:19,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741834_1010 (size=25344) 2024-11-13T18:39:19,827 DEBUG [RS:0;39e84130bbc9:35235 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/oldWALs 2024-11-13T18:39:19,828 INFO [RS:0;39e84130bbc9:35235 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39e84130bbc9%2C35235%2C1731522975587.meta:.meta(num 1731522977959) 2024-11-13T18:39:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741833_1009 (size=15059713) 2024-11-13T18:39:19,831 DEBUG [RS:0;39e84130bbc9:35235 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/oldWALs 2024-11-13T18:39:19,831 INFO [RS:0;39e84130bbc9:35235 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39e84130bbc9%2C35235%2C1731522975587:(num 1731522977467) 2024-11-13T18:39:19,831 DEBUG [RS:0;39e84130bbc9:35235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:39:19,831 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:39:19,831 INFO [RS:0;39e84130bbc9:35235 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:39:19,831 INFO [RS:0;39e84130bbc9:35235 {}] hbase.ChoreService(370): Chore service for: regionserver/39e84130bbc9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T18:39:19,831 INFO [RS:0;39e84130bbc9:35235 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:39:19,831 INFO [regionserver/39e84130bbc9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:39:19,832 INFO [RS:0;39e84130bbc9:35235 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35235 2024-11-13T18:39:19,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39e84130bbc9,35235,1731522975587 2024-11-13T18:39:19,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:39:19,835 INFO [RS:0;39e84130bbc9:35235 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:39:19,837 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39e84130bbc9,35235,1731522975587] 2024-11-13T18:39:19,839 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39e84130bbc9,35235,1731522975587 already deleted, retry=false 2024-11-13T18:39:19,839 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 39e84130bbc9,35235,1731522975587 expired; onlineServers=0 2024-11-13T18:39:19,839 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '39e84130bbc9,36105,1731522974847' ***** 2024-11-13T18:39:19,839 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T18:39:19,839 INFO [M:0;39e84130bbc9:36105 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:39:19,839 INFO [M:0;39e84130bbc9:36105 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:39:19,839 DEBUG [M:0;39e84130bbc9:36105 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T18:39:19,840 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T18:39:19,840 DEBUG [M:0;39e84130bbc9:36105 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T18:39:19,840 DEBUG [master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.large.0-1731522977096 {}] cleaner.HFileCleaner(306): Exit Thread[master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.large.0-1731522977096,5,FailOnTimeoutGroup] 2024-11-13T18:39:19,840 DEBUG [master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.small.0-1731522977097 {}] cleaner.HFileCleaner(306): Exit Thread[master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.small.0-1731522977097,5,FailOnTimeoutGroup] 2024-11-13T18:39:19,840 INFO [M:0;39e84130bbc9:36105 {}] hbase.ChoreService(370): Chore service for: master/39e84130bbc9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T18:39:19,840 INFO [M:0;39e84130bbc9:36105 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:39:19,840 DEBUG [M:0;39e84130bbc9:36105 {}] master.HMaster(1795): Stopping service threads 2024-11-13T18:39:19,840 INFO [M:0;39e84130bbc9:36105 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T18:39:19,840 INFO [M:0;39e84130bbc9:36105 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:39:19,840 ERROR [M:0;39e84130bbc9:36105 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T18:39:19,841 INFO [M:0;39e84130bbc9:36105 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:39:19,841 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T18:39:19,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T18:39:19,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:39:19,842 DEBUG [M:0;39e84130bbc9:36105 {}] zookeeper.ZKUtil(347): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T18:39:19,842 WARN [M:0;39e84130bbc9:36105 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T18:39:19,842 INFO [M:0;39e84130bbc9:36105 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/.lastflushedseqids 2024-11-13T18:39:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742490_1666 (size=72) 2024-11-13T18:39:19,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:39:19,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35235-0x100ed62c5980001, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:39:19,939 INFO [RS:0;39e84130bbc9:35235 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:39:19,939 INFO [RS:0;39e84130bbc9:35235 {}] regionserver.HRegionServer(1031): Exiting; stopping=39e84130bbc9,35235,1731522975587; zookeeper connection closed. 2024-11-13T18:39:19,939 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@79a52f48 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@79a52f48 2024-11-13T18:39:19,940 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T18:39:20,251 INFO [M:0;39e84130bbc9:36105 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:39:20,252 INFO [M:0;39e84130bbc9:36105 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T18:39:20,252 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:39:20,252 INFO [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:39:20,252 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:39:20,252 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:39:20,252 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:39:20,252 INFO [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=786.54 KB heapSize=969.44 KB 2024-11-13T18:39:20,268 DEBUG [M:0;39e84130bbc9:36105 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe7fc0126b084263ab11f0aa1abc88ee is 82, key is hbase:meta,,1/info:regioninfo/1731522978062/Put/seqid=0 2024-11-13T18:39:20,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742491_1667 (size=5672) 2024-11-13T18:39:20,672 INFO [M:0;39e84130bbc9:36105 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2265 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe7fc0126b084263ab11f0aa1abc88ee 2024-11-13T18:39:20,692 DEBUG [M:0;39e84130bbc9:36105 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fdca5020692410f8ee51764fa0ecec3 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1731523132078/Put/seqid=0 2024-11-13T18:39:20,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742492_1668 (size=44827) 2024-11-13T18:39:21,095 INFO [M:0;39e84130bbc9:36105 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785.99 KB at sequenceid=2265 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fdca5020692410f8ee51764fa0ecec3 2024-11-13T18:39:21,098 INFO [M:0;39e84130bbc9:36105 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fdca5020692410f8ee51764fa0ecec3 2024-11-13T18:39:21,113 DEBUG [M:0;39e84130bbc9:36105 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29e0b04b0dec45818d256023d7670258 is 69, key is 39e84130bbc9,35235,1731522975587/rs:state/1731522977217/Put/seqid=0 2024-11-13T18:39:21,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073742493_1669 (size=5156) 2024-11-13T18:39:21,517 INFO [M:0;39e84130bbc9:36105 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2265 (bloomFilter=true), to=hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29e0b04b0dec45818d256023d7670258 2024-11-13T18:39:21,521 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe7fc0126b084263ab11f0aa1abc88ee as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe7fc0126b084263ab11f0aa1abc88ee 2024-11-13T18:39:21,524 INFO [M:0;39e84130bbc9:36105 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe7fc0126b084263ab11f0aa1abc88ee, entries=8, sequenceid=2265, filesize=5.5 K 2024-11-13T18:39:21,525 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fdca5020692410f8ee51764fa0ecec3 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6fdca5020692410f8ee51764fa0ecec3 2024-11-13T18:39:21,529 INFO [M:0;39e84130bbc9:36105 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fdca5020692410f8ee51764fa0ecec3 2024-11-13T18:39:21,529 INFO [M:0;39e84130bbc9:36105 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6fdca5020692410f8ee51764fa0ecec3, entries=174, sequenceid=2265, filesize=43.8 K 2024-11-13T18:39:21,530 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29e0b04b0dec45818d256023d7670258 as hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29e0b04b0dec45818d256023d7670258 2024-11-13T18:39:21,533 INFO [M:0;39e84130bbc9:36105 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33577/user/jenkins/test-data/fc81b5cf-6454-b48b-5939-c3941c9fa462/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29e0b04b0dec45818d256023d7670258, entries=1, sequenceid=2265, filesize=5.0 K 2024-11-13T18:39:21,534 INFO [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(3140): Finished flush of dataSize ~786.54 KB/805418, heapSize ~969.14 KB/992400, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2265, compaction requested=false 2024-11-13T18:39:21,538 INFO [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:39:21,538 DEBUG [M:0;39e84130bbc9:36105 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731523160252Disabling compacts and flushes for region at 1731523160252Disabling writes for close at 1731523160252Obtaining lock to block concurrent updates at 1731523160252Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731523160252Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=805418, getHeapSize=992640, getOffHeapSize=0, getCellsCount=2464 at 1731523160252Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731523160253 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731523160253Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731523160268 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731523160268Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731523160675 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731523160691 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731523160691Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731523161099 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731523161113 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731523161113Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@475dc17d: reopening flushed file at 1731523161520 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75f165d: reopening flushed file at 1731523161524 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@427db31f: reopening flushed file at 1731523161529 (+5 ms)Finished flush of dataSize ~786.54 KB/805418, heapSize ~969.14 KB/992400, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2265, compaction requested=false at 1731523161534 (+5 ms)Writing region close event to WAL at 1731523161538 (+4 ms)Closed at 1731523161538 2024-11-13T18:39:21,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42373 is added to blk_1073741830_1006 (size=953858) 2024-11-13T18:39:21,542 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:39:21,542 INFO [M:0;39e84130bbc9:36105 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T18:39:21,542 INFO [M:0;39e84130bbc9:36105 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36105 2024-11-13T18:39:21,542 INFO [M:0;39e84130bbc9:36105 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:39:21,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:39:21,646 INFO [M:0;39e84130bbc9:36105 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:39:21,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36105-0x100ed62c5980000, quorum=127.0.0.1:53425, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:39:21,651 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:39:21,653 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:39:21,653 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:39:21,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:39:21,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/hadoop.log.dir/,STOPPED} 2024-11-13T18:39:21,656 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:39:21,656 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:39:21,656 WARN [BP-941627117-172.17.0.3-1731522971541 heartbeating to localhost/127.0.0.1:33577 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:39:21,656 WARN [BP-941627117-172.17.0.3-1731522971541 heartbeating to localhost/127.0.0.1:33577 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-941627117-172.17.0.3-1731522971541 (Datanode Uuid 03a43742-4062-494e-8538-b44d5612c322) service to localhost/127.0.0.1:33577 2024-11-13T18:39:21,659 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/cluster_a648365b-8906-3617-8e5f-6b51b5e809af/data/data1/current/BP-941627117-172.17.0.3-1731522971541 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:39:21,660 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/cluster_a648365b-8906-3617-8e5f-6b51b5e809af/data/data2/current/BP-941627117-172.17.0.3-1731522971541 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:39:21,660 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:39:21,668 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:39:21,668 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:39:21,668 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:39:21,668 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:39:21,669 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb85f69e-68a0-7f8e-f5c1-0eb4b32ae687/hadoop.log.dir/,STOPPED} 2024-11-13T18:39:21,686 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T18:39:21,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down